{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "MjmWt71VW4NS"
   },
   "source": [
    "# Transformer 模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 127
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 369266,
     "status": "ok",
     "timestamp": 1581684635265,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "VhidSmlFum3_",
    "outputId": "d59ebdb7-7a84-4bfd-ce4c-ef47611b5091"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 107
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 11189,
     "status": "ok",
     "timestamp": 1581684779367,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "hsmSDBrhXJls",
    "outputId": "4dc696e8-f3a2-4232-de03-5d6145fc2e17"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorFlow 2.x selected.\n",
      "WARNING:tensorflow:From <ipython-input-3-b2ae02e03e62>:3: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.config.list_physical_devices('GPU')` instead.\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "%tensorflow_version 2.x\n",
    "import tensorflow as tf\n",
    "print(tf.test.is_gpu_available())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 161
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2911,
     "status": "ok",
     "timestamp": 1581684783643,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "6qr1xukGW4NX",
    "outputId": "5c03c488-dfe9-4364-e5f9-ec5b1f88cada"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1.0\n",
      "sys.version_info(major=3, minor=6, micro=9, releaselevel='final', serial=0)\n",
      "matplotlib 3.1.3\n",
      "numpy 1.17.5\n",
      "pandas 0.25.3\n",
      "sklearn 0.22.1\n",
      "tensorflow 2.1.0\n",
      "tensorflow_core.python.keras.api._v2.keras 2.2.4-tf\n"
     ]
    }
   ],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "\n",
    "print(tf.__version__)\n",
    "print(sys.version_info)\n",
    "for module in mpl,np,pd,sklearn,tf,keras:\n",
    "    print(module.__name__,module.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_Pr1yRWVW4Nf"
   },
   "outputs": [],
   "source": [
    "# gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "# for gpu in gpus:\n",
    "#     tf.config.experimental.set_memory_growth(gpu, True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "jYZcx7m0W4Ni"
   },
   "source": [
    "步骤：\n",
    "1. loads data\n",
    "2. preprocessess data -> dataset\n",
    "3. tools\n",
    "   - generates position embedding\n",
    "   - create mask (a. padding, b. decoder)\n",
    "   - scaled_dot_product_attention\n",
    "4. build model\n",
    "   - MultiheadAttention\n",
    "   - EncoderLayer\n",
    "   - DecoderLayer\n",
    "   - EncoderModel\n",
    "   - DecoderModel\n",
    "   - Transformer\n",
    "5. train\n",
    "   - initializes model\n",
    "   - define loss, optimizer, learning_rate schedule\n",
    "   - train_step\n",
    "   - train process\n",
    "6. train step -> train process\n",
    "7. Evaluate and Visualize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "aCX9sjG4W4Nk"
   },
   "source": [
    "## 数据处理\n",
    "### 加载数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 857,
     "referenced_widgets": [
      "b20f695e03d748bdb2016a09c6712e6a",
      "68c2817c35724d908b2dc27ddbc8d65a",
      "edf5019ce88945bd955f66e539a68c55",
      "e492e498b2da4c3c8be865af31caeac8",
      "84e50d0a4e824d409249d1c05f13a8fc",
      "792f241ea8394e9ca81eeac6909481f8",
      "f6ca90f14db146c0a44e93674585786d",
      "66669c75d14146139cba84f57a4c12ac",
      "ea263382231d42b29888fb3d1b16acca",
      "3dbe4b29ee9d467cbeef39946925db10",
      "6276134a3fa7469faa2cdb9b9841a236",
      "a74e3d4851ad40a9802864c7e9fa3313",
      "1f0d1a90ce324a4c946574e4dc1affba",
      "b234ad53c4f048b3b336d9a3e5441189",
      "1a1013a121c6452a9dcf5be7d91b6bb6",
      "e44536ead67141a58ce0aaad0ede93d2",
      "918b595ca82f4e889e8851e7a3301542",
      "06d9c755ed0e4758a602472ba8a7c886",
      "db2ba1f0367d4dc4abcea8e19736a80f",
      "e1050782a98a48c7ba72adacd240b84a",
      "d769fc5c617e4a9dbd5ae1b132b9981b",
      "46ec15c44cf0445eb7097b9ffcd48b72",
      "4469976e9ed0411eb472ce953b2b942f",
      "c76e58b8968747bc915fc37594da12fb",
      "8e4d74fcb67d49378495b45510ff668c",
      "985545ca832b43fbb75382850f1c39f1",
      "00d1991457d541d48c1ca5aad7fe3e8f",
      "8636cf713e6548aca61d10c821199873",
      "9ebf269ea639433484a26cffb896048f",
      "365509bb19e3492ca4cf0467a65677cf",
      "89a784492dfb49a1a83aa693ef82a7b1",
      "39e1c098df36444c932002921ed9bd51",
      "c6c4fa9e09454d6f81d9e7cabc1ad9d8",
      "a845cf55d3cc451cbf6023f53451b987",
      "e522f0058696468a979a4232787e8057",
      "0e04ea39154b40819a6ae5b5914847a2",
      "056d566b9dfc4c12b13b0fcb6075a270",
      "bd7d5597bcc24a5ba6ed233ff6869af3",
      "21fec75ed3724763aa3cd93b6e2c7ab9",
      "e65cf9ab4221419e97f74426d0ee292f",
      "ed49da325e7146d5bddbd6b2ee6e338b",
      "2b125be6a55f43798366013f4d4ee840",
      "de36871931e84fef867700db40d0126c",
      "2966a7701dde4f46885083de9c6b1eb0",
      "02fa22f70da849ceb5cef6c6a9a6c42f",
      "d0979a4394f84ddb89c4307a18df05ee",
      "1a6d255cf1744383b9d216d4a04f0acf",
      "84890da093d9455b8bfc29a6f432bc8f",
      "34171b17684e49708e0633eed1c8216c",
      "d8b58951df5c4dfeb6a1d24669b522ff",
      "ed874e7a5d294e28940f3ac0dcc99a96",
      "9bf6fd00b1b64af2ba4b1574f0e66bc0",
      "ac26758ef636439ea359ddf0d1729743",
      "5599b46c0a374da4a1bc3566747f395c",
      "90a547d5b93746ceae81fd001be9004f",
      "24187cb2ff0e45f9833a8c48c96efcc7",
      "8543fc0a11d841bb9593d942eaf05601",
      "c003c5f9ade94009a41a9d225bc12502",
      "eaa4b7905b524f5ea5e2d5c73437e0d5",
      "76125a19d0e24b21a3e0cae93fe34b60",
      "f3d20f2e66ed4f059a3a26ffe703adf3",
      "245051a8a9e746568bc24399fded1713",
      "80bca50374f8438bacd78207ad6c8e39",
      "2121f86cde9c4eefb6b465d07b9e01df",
      "8a66272607e04f92b935580029abe3a1",
      "d2077508138b42e688b069d59ba044a0",
      "5d733f7bd7f245ce8e92cd7a74c9a967",
      "66234e0238854b9cb36cb57a1f2aa44f",
      "7dc0f107d2be48de8be39dfbaff50c64",
      "b1f9c517f8404821aa7e374d674982e8",
      "5ffbbf72f5dd402ca28cd2d35d79295b",
      "bb6f1ba6e0cf46889984cfeb9bd16097"
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 24988,
     "status": "ok",
     "timestamp": 1581684807716,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "O7oH9djwW4Nl",
    "outputId": "1527315d-606b-4d3b-c68e-efc32e2b4c49"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1mDownloading and preparing dataset ted_hrlr_translate (124.94 MiB) to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0...\u001b[0m\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b20f695e03d748bdb2016a09c6712e6a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', description='Dl Completed...', max=1, style=ProgressStyl…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ea263382231d42b29888fb3d1b16acca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', description='Dl Size...', max=1, style=ProgressStyle(des…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "918b595ca82f4e889e8851e7a3301542",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', description='Extraction completed...', max=1, style=Prog…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8e4d74fcb67d49378495b45510ff668c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shuffling and writing examples to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0.incomplete6QGB3C/ted_hrlr_translate-train.tfrecord\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6c4fa9e09454d6f81d9e7cabc1ad9d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=51785), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed49da325e7146d5bddbd6b2ee6e338b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shuffling and writing examples to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0.incomplete6QGB3C/ted_hrlr_translate-validation.tfrecord\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34171b17684e49708e0633eed1c8216c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1193), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8543fc0a11d841bb9593d942eaf05601",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shuffling and writing examples to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0.incomplete6QGB3C/ted_hrlr_translate-test.tfrecord\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a66272607e04f92b935580029abe3a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1803), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1mDataset ted_hrlr_translate downloaded and prepared to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0. Subsequent calls will reuse this data.\u001b[0m\n",
      "tfds.core.DatasetInfo(\n",
      "    name='ted_hrlr_translate',\n",
      "    version=1.0.0,\n",
      "    description='Data sets derived from TED talk transcripts for comparing similar language pairs\n",
      "where one is high resource and the other is low resource.\n",
      "',\n",
      "    homepage='https://github.com/neulab/word-embeddings-for-nmt',\n",
      "    features=Translation({\n",
      "        'en': Text(shape=(), dtype=tf.string),\n",
      "        'pt': Text(shape=(), dtype=tf.string),\n",
      "    }),\n",
      "    total_num_examples=54781,\n",
      "    splits={\n",
      "        'test': 1803,\n",
      "        'train': 51785,\n",
      "        'validation': 1193,\n",
      "    },\n",
      "    supervised_keys=('pt', 'en'),\n",
      "    citation=\"\"\"@inproceedings{Ye2018WordEmbeddings,\n",
      "      author  = {Ye, Qi and Devendra, Sachan and Matthieu, Felix and Sarguna, Padmanabhan and Graham, Neubig},\n",
      "      title   = {When and Why are pre-trained word embeddings useful for Neural Machine Translation},\n",
      "      booktitle = {HLT-NAACL},\n",
      "      year    = {2018},\n",
      "      }\"\"\",\n",
      "    redistribution_info=,\n",
      ")\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import tensorflow_datasets as tfds\n",
    "\n",
    "examples, info = tfds.load('ted_hrlr_translate/pt_to_en',\n",
    "                           with_info = True, as_supervised = True)\n",
    "train_examples, val_examples = examples['train'], examples['validation']\n",
    "\n",
    "print(info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 287
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 24766,
     "status": "ok",
     "timestamp": 1581684807717,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "FddiHkokW4Np",
    "outputId": "bdd68b77-dc17-4717-9eeb-88cbd46237ec"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'e quando melhoramos a procura , tiramos a \\xc3\\xbanica vantagem da impress\\xc3\\xa3o , que \\xc3\\xa9 a serendipidade .'\n",
      "b'and when you improve searchability , you actually take away the one advantage of print , which is serendipity .'\n",
      "\n",
      "b'mas e se estes fatores fossem ativos ?'\n",
      "b'but what if it were active ?'\n",
      "\n",
      "b'mas eles n\\xc3\\xa3o tinham a curiosidade de me testar .'\n",
      "b\"but they did n't test for curiosity .\"\n",
      "\n",
      "b'e esta rebeldia consciente \\xc3\\xa9 a raz\\xc3\\xa3o pela qual eu , como agn\\xc3\\xb3stica , posso ainda ter f\\xc3\\xa9 .'\n",
      "b'and this conscious defiance is why i , as an agnostic , can still have faith .'\n",
      "\n",
      "b\"`` `` '' podem usar tudo sobre a mesa no meu corpo . ''\"\n",
      "b'you can use everything on the table on me .'\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 输出test\n",
    "for pt, en in train_examples.take(5):\n",
    "    print(pt.numpy())\n",
    "    print(en.numpy())\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WSyoqbXeW4Ns"
   },
   "outputs": [],
   "source": [
    "# 从语料中构建subwords-level\n",
    "en_tokenizer = tfds.features.text.SubwordTextEncoder.build_from_corpus(\n",
    "    (en.numpy() for pt, en in train_examples), target_vocab_size = 2 ** 13) # 2192\n",
    "pt_tokenizer = tfds.features.text.SubwordTextEncoder.build_from_corpus(\n",
    "    (pt.numpy() for pt, en in train_examples), target_vocab_size = 2 ** 13)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 179
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 144969,
     "status": "ok",
     "timestamp": 1581684928318,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "L2_mry3nW4Nw",
    "outputId": "f694b777-ee31-411a-fccc-63f6870c1733"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokenized string is [7915, 1248, 7946, 7194, 13, 2799, 7877]\n",
      "The original string is Transformer is awesome.\n",
      "7915 --> \"T\"\n",
      "1248 --> \"ran\"\n",
      "7946 --> \"s\"\n",
      "7194 --> \"former \"\n",
      "13 --> \"is \"\n",
      "2799 --> \"awesome\"\n",
      "7877 --> \".\"\n"
     ]
    }
   ],
   "source": [
    "# test \n",
    "sample_string = 'Transformer is awesome.'\n",
    "\n",
    "tokenized_string = en_tokenizer.encode(sample_string)\n",
    "print('Tokenized string is {}'.format(tokenized_string))\n",
    "\n",
    "origin_string = en_tokenizer.decode(tokenized_string)\n",
    "print('The original string is {}'.format(origin_string))\n",
    "\n",
    "assert origin_string == sample_string\n",
    "\n",
    "for token in tokenized_string:\n",
    "    print('{} --> \"{}\"'.format(token, en_tokenizer.decode([token])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "U2BW9ifsW4N0"
   },
   "source": [
    "### 构建Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zKBE_OQBW4N1"
   },
   "outputs": [],
   "source": [
    "buffer_size = 20000\n",
    "batch_size = 64\n",
    "max_length = 40\n",
    "\n",
    "# 把句子转为subword形式\n",
    "def encode_to_subword(pt_sentence, en_sentence):\n",
    "    pt_sentence = [pt_tokenizer.vocab_size] + pt_tokenizer.encode(\n",
    "        pt_sentence.numpy()) + [pt_tokenizer.vocab_size + 1]\n",
    "    en_sentence = [en_tokenizer.vocab_size] + en_tokenizer.encode(\n",
    "        en_sentence.numpy()) + [en_tokenizer.vocab_size + 1]\n",
    "    return pt_sentence, en_sentence\n",
    "\n",
    "# 过滤大于max_length的句子\n",
    "def filter_by_max_length(pt, en):\n",
    "    return tf.logical_and(tf.size(pt) <= max_length, tf.size(en) <= max_length)\n",
    "\n",
    "# 使用tf.py_function封装py函数\n",
    "def tf_encode_to_subword(pt_sentence, en_sentence):\n",
    "    return tf.py_function(encode_to_subword, [pt_sentence, en_sentence], [tf.int64, tf.int64])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yctnVUQwW4N4"
   },
   "outputs": [],
   "source": [
    "train_dataset = train_examples.map(tf_encode_to_subword)\n",
    "train_dataset = train_dataset.filter(filter_by_max_length)\n",
    "train_dataset = train_dataset.shuffle(\n",
    "    buffer_size).padded_batch(batch_size, padded_shapes=([-1], [-1]))\n",
    "\n",
    "valid_dataset = val_examples.map(tf_encode_to_subword)\n",
    "valid_dataset = valid_dataset.filter(\n",
    "    filter_by_max_length).padded_batch(batch_size, padded_shapes=([-1], [-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 107
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 268657,
     "status": "ok",
     "timestamp": 1581685052904,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "m2tV25CoW4N9",
    "outputId": "1969d93a-d99f-4af9-93d0-7bcf83d0587c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 38) (64, 40)\n",
      "(64, 39) (64, 35)\n",
      "(64, 39) (64, 39)\n",
      "(64, 39) (64, 39)\n",
      "(64, 39) (64, 36)\n"
     ]
    }
   ],
   "source": [
    "# test\n",
    "for pt_batch, en_batch in valid_dataset.take(5):\n",
    "    print(pt_batch.shape, en_batch.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "iwiSL9FeW4OA"
   },
   "source": [
    "## 工具函数\n",
    "### 位置编码\n",
    "根据公式写函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kXnuYH0sW4OB"
   },
   "outputs": [],
   "source": [
    "# 求角度\n",
    "# pos.shape: [setence_length, 1]\n",
    "# i.shape: [1, d_model]\n",
    "# result.shape: [setence_length, d_model]\n",
    "def get_angles(pos, i, d_model):\n",
    "    angle_rates = 1 / np.power(10000, (2 * (i // 2)) / np.float32(d_model))\n",
    "    return pos * angle_rates\n",
    "\n",
    "# 求编码\n",
    "def get_position_embedding(setence_length, d_model):\n",
    "    angle_rads = get_angles(np.arange(setence_length)[:, np.newaxis],\n",
    "                            np.arange(d_model)[np.newaxis, :], d_model)\n",
    "\n",
    "    # angle_rads.shape: [setence_length, d_model / 2]\n",
    "    angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n",
    "    angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n",
    "    \n",
    "    # position_embedding.shape: [setence_length, d_model]\n",
    "    # position_embedding = np.concatenate([sines, cosines], axis = -1)\n",
    "    # position_embedding.shape: [1, setence_length, d_model]\n",
    "    position_embedding = angle_rads[np.newaxis, ...]\n",
    "    return tf.cast(position_embedding, dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1556,
     "status": "ok",
     "timestamp": 1581687621393,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "8oDsjTjQW4OE",
    "outputId": "16d4003f-5009-4989-de96-24928a03263f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 50, 512)\n"
     ]
    }
   ],
   "source": [
    "# test\n",
    "position_embedding = get_position_embedding(50, 512)\n",
    "print(position_embedding.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 283
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1972,
     "status": "ok",
     "timestamp": 1581687624274,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "75K0v7DvW4OI",
    "outputId": "7ec28dc7-1273-4da1-827a-c585a3ee97cd"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEKCAYAAAD+XoUoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOydd5xU1fmHn/femdneKywsvVooIojY\nsHeN3cRYYjSJ5afGaDSJJjHFmKIxicagMdEUe0zAYLCgoqCAhY60pe7Csn13dqfdO+f3x70zO7ss\nMMAusHiez+c4t98z63Dmzvc97/cVpRQajUaj+WJgHOgOaDQajWb/oQd9jUaj+QKhB32NRqP5AqEH\nfY1Go/kCoQd9jUaj+QKhB32NRqP5AtGjg76IbBCRpSKySEQ+drfli8ibIrLGfc3ryT5oNBrNgUJE\nnhaR7SKybCf7RUR+JyJrRWSJiIxP2HeNO06uEZFruqtP++NJf6pSaqxSaoK7fg/wtlJqGPC2u67R\naDSHIn8FztzF/rOAYW67EfgjOA/HwA+BScBE4Ifd9YB8IOSdC4Bn3OVngAsPQB80Go2mx1FKzQHq\nd3HIBcCzyuEjIFdE+gBnAG8qpeqVUg3Am+z6yyNpPN1xkV2ggDdERAF/UkpNA0qUUlvd/duAkq5O\nFJEbcb75yEhPO6pNpTN21AAWrdzI2JHlbP5sOQMOG8SiTU1k5OXQr2UrDY0hSscdxpI1W/GmpXNY\noYmyLVa1eGhrqKOobwllqomq9TWkGkLhyIGsbxUattdhen0UFuVSU12HikbJKshnSEEaoc0V1NcF\nsBXkpnvJLC+hzZvNhuoWSvLTKUgBq2YbrdtbaLGiAKSbBhm5KaQUFWCn5VD52XJ8ImSkmKTmpuHN\nyyOamkVL2KahNUxbwMIKBbEjYYjajB9SRLS1mXBLG5HWMOFwlFBUYStFFBDAFPCIUFCWixUIYQUt\n7JBNOBrFihI/NpZvbQDZh40kbCvCVpSwZRO2okRt5bRoFBW13eYsjyn1IR4vYnrBMFGG6bwiRBXY\nCpRy+rWqYisiAiII7qthtK8bBiIGIoI3xUQpQCmUew1nHZTzH2KZ4kpFycxKRUQQwBDBvQ2CYAjO\nPndbVWVd/F0r5wKdPpHt60MG9UFinzf3P+KudVx3WLl2SzKfeQAOH9a/y+0iO25bumpT0tcFOHJk\nedfX7mLb4s+Tv/bYnVy3KxbtwXWdaw/Yg2tvTP66ozped9HKjahAXa1Sqijpi3TCyO6nsIJJHasC\ndcuBxIOnueNcspQBmxPWt7jbdrZ9n+npQf84pVSliBQDb4rI54k7lVLK/ULYAfcPNw3gqCMPU0vN\nScyd+zg5k29izgeP8Z2MUTz28pMU3DKLYy85mwdn/4RXZ6zhu3Pn0vf8Byk97CjmX5+B3VTHie8V\n8MlL/+CK++/gwfBr/PirTzI808e1Lz/JVxek8upjfyWzdCDX3ngOf3zkn0SCrRx/9WW8/JXDWX/b\nV/nH35fSFIly0ahSjv39d1hcdjJXP/I+d145hqsHm9Q+8TPm/2EO79S0ATA+J5VJ5w1jyI3X0HL4\nWXw/ezR9UzxMHpjDiAuPpO8ll9A68mTe29jE8x9vZsmSaravW41/2wasoJ8FL91I2/w3qHxvEVUL\nK9m4qZkNbRHqwzbhqMIUyPGaFPpMrrn7AmqXrKNuVS0NFY1U+sPUhGwaIjYBO4rt/nV9hnDWy2+w\nqSnIhtpWNta1UlXXRmtziLamEMG2MKGWRsJtTVgBP1awlbnfKccsKMXMK4aMXKIpWUTTcomYKbRF\norRGogQsRXPI4uQrfoTp9WF4fBgeL4bHh5mShunxxZcNjw+Pz0u/YQVY4ShWxMaK2NhWFCsSJWpF\nse0othUlakexLYuoFWbySSPweQx8HtN5NQ1SPIa7rWO774d/RUVt5zPkfnk5y85r1H0FePwv38MQ\nMEUwRDAN50ul87oIGAhHnX93h2vtihlvPAK0D/Kxn9TibjASRugBU2/d7fUSefu9P3Q5wBtdbCw+\n/pakr/veB491WO/qHjHyp9yc9HUBPpj7eNLH5h57U9LHzu103ZzJNxFZ9JfkvzW6wgriGXF+UodG\nFv0lmCBd9wp6VN5RSlW6r9uBV3G0qWr35wvu6/ae7INGo9HsESKIYSbVuoFKIPFnYT9328627zM9\nNuiLSIaIZMWWgdOBZcB0IBaJvgb4T0/1QaPRaPYccX+x7r51A9OBq91ZPMcATa78PQs4XUTy3ADu\n6e62faYn5Z0S4FX356wH+KdS6n8ishB4UUSuBzYCl/VgHzQajWbPcJ/0u+dS8hxwElAoIltwZuR4\nAZRSTwAzgbOBtUAbcJ27r15EfgIsdC/1gFJqVwHhpOmxQV8pVQGM6WJ7HXDKnlxrxfYwk++6mndH\nTmLyrY/y0dEncNkRxVw2z/mmnX5eHrfdtJIf/OwczvrjfIJNtTx7x/G8c9bpRF55jSUzf0H55HN5\n6IzBvD3iOQK24rRvTGZ+6mjee/0VVNRm9AlH8/LMVbTVVTHg2PO4+9ThRN98isXTV1MTshmfm8qo\nyyZgjT2Hv/9vDePG9eH0oQVEF/yTDW8uZ2lTiHBU0T/Ny+CheZSdMBaGTWJFTYBMj8GgDC/FRxRT\nOOEwVPkRbGoO88nmRtZvaaa5toFgQzVW0A9AuGI5jas307i+gcatfmpCNn4rSjjqCPQ+Q8gwDfJ9\nJq2VtbRt99NWG6ApaOG3orTazrExPd8UpzUEIjS0halrDVPnDxMKWIQDFuGQRSTYhh0OYIcCRK0w\nKmpjpGdhpGYgvjSinlSULx3lSSFsKScgHFWE7SghK4qYsZ+8BmKYGF4fhvsT2PD4EMPE9HgQEeyY\ndm9HUVEnkKyiiqhyXpVSRKMqrp2bhmAahvMq4q530RKipCoa3fXn03avnaSe337d3ev5e0JXgd29\noSs9X/bh4t3UrV6JAGJ2z6CvlLpyN/sV0GWARCn1NPB0t3QkgZ4O5Go0Gk3vQgSjm570D0b0oK/R\naDSd6C5552BED/oajUaTSDdq+gcjetDXaDSaBATB8HgPdDd6jF7hshlqaeTtk4O8saWZ2eeYvLqy\nhmPmz+G1x57ipz+5nrdO+DJH56VRe+3Pmf/8i0y87FJGz32MV1fWcPtjH6Jsm/uuP5rah25nZmUz\nZ/XPps+3f8x3X1pC7eqFFB82hfvOP4zKT98ho6g/Z586lGPSG1k+7TUWNgTJ8RqMn1xG0UVf4a31\njby3cDNXTOhPWet6Kl+fzeplNVSHLNJMYXS2j37HDiRj0slsM3KZu7GekhQP/QfmUjphKKlHTKbB\nV8CirS18urGB+q0ttG7fRLi1CQDTl0Zwwzoa11bRvKWZmpBNs+UkWoETxM30GOR43UDutjr81a20\n1QdoikTjAV87IfPUFMFnCPXBCNubQ9T5QwQDEcKBCOGQ5SRIhQJYYSeIG7Ui2JEwRkY2RkYWUV8a\nUW8aypNCROEEcKMKy4a2iE3QisaDtrEgriQGcc1YMFcwPQYqihOwjSpsO+oGbVU8OUu5QVwVtVG2\nHQ/U+kwnASuWmNU5kGuIdAi07ioxC7oOfu6MPYmJxu6XTGLW3vBFDrLuF/bvPP39jn7S12g0mk70\n1gE9GfSgr9FoNImIdNuUzYMRPehrNBpNAsKh/aTfKzT9/uV9+Pmxt3D/7y/n4QnXc9ddJ3L099+k\nz7hTub763/y7ooEvT/8xF/10NukFfXntW5N4/ua/MzwzhfUfTOfIc87nqvwaZjz6Pvk+kxMevJRn\n1iuWv/0+vowcTjvzcE5Kr8UOBxg8aTK3HT+Qxuf+wEdzt+C3ohyTn8aor06lKv9wnp63gaqVnzN1\nYA6BOa+y/q11rPaHsRUMTPfRf3wpfaYegzXgKD6pauGdldsZmumlz/g+5IwdS6TPYaxtCPLxxgaq\nNjfRsn0rYX8DUSuMGCa+jBwaVm+maWMz9XWBeGJWonFaLDErPT+Nlq1+2uoC1IdtmiI2QVdvT0zM\n8hlCmmlQ7w9T3xqmMZaYFbKJhCysgB87HCAacfX8WHJWRhbKl4HypoM3lagnhWAsMctWBK0oQStK\nW8TuYLQmhomRkJRleHwYhmCaBqZpxBOzbCuKihLX8uPafkzTtx1dP2a0ZhqCJ0HD76Dri2C6Ynd3\nJ2ZJ/LrJJ2btTM/v6ph9RSdmdTNiYHp8SbXeiH7S12g0mkTk0H7S14O+RqPRJCDoefoajUbzheJQ\nHvR7haaf3VBJaaqHx4d/DYAl1zzEmndeZfYvzubRrzzO1SeU86g1no3zZvDtOy+l6vavsLAhyJX3\nn0l2v+H89YaJfHbzXSxuCnLByQNpPfsOfvPcYvzVGxh4zFR+cOpQtv7x1xQMHc+3zhtFeeWHLH7q\nA1a2hOif5uXwL43Cd+rV/HtVDcs/20rzltWkrXmfdf/5kCWbmqgP2+T7TEaWZtD/pNH4xk1lbbPi\n3TW1VFY00PeIYkonjcYz+hgqQyafbm1m8YZ66qv9BBq2xefoe9IySckppHFtNc1bmtkWjM3Rbzda\ny/Q4en5OqoeMknRaq1tpaQrRFIkSjCoCdrsxW+wcnyGkGhKfox8KWIQCESIhi0gwiB125ujb7jz9\nmJYuaVkoXxrKm0LUm0bIisb1/LCtaIvYtEWihOxo3GgtZrwW1/PdOfuGaWB4DMQQopZTMEUp5RRM\n6WS0FjN8i7XdGq25c/QNQ+J6/u7m6MfYmZ7fmWTn1u9O949d52DV8w80B0XX9Tx9jUaj+SKh5R2N\nRqP5wiAiGN7eOTMnGfSgr9FoNIlowzWNRqP5YqEH/QPMtmo/1237mOzzf41/wTQKb/8jU2+4ntZb\nL6fVjjL+9dc558JfMuSkC7mnTxXf++siLh5ZQPBrP+OKQRWUz3mCH7+1nqPzUhn38I+4dsZKNsyb\nRU75KG65+HDKVv6X6U9+xNgfX89Vhxey7vY7+KCiAVOEY0fkM+Cqy1gUyOK59z6jZtUnRK0w22e8\nSsWcTWwORPAZwvBMH+VT+pF3/Ek05g7hgxU1fLyqhobKKvpMGEjG2MkE8gezbEMT89bUUlvZQmvN\nJkItDU4ilMeHLz2b9IIyGj9porolTEOkPYhrCmR6DLLdQG5GSQaZJRnUr2mgPuwkcCVW14KOQdw0\n06C+NURLa5hQMEI4YBEJWe1Ga25iVjQhgKp8jsma8qZjYRC2o25zgrghO0rIsp3KWQkGa2anIK7p\nMTA9hhMo9RjxRCzbUnHjtVhiVqLRWodAbqfErA4JWm5iVqxy1q6CuLHELOg6YBsjMTFrb4O43W20\ntj/oBV3cLxi94X/WXtIrZu9oNBrN/kJEECO5luT1zhSRVSKyVkTu6WL/IyKyyG2rRaQxYZ+dsG96\nd7y/XvGkr9FoNPsT0+ye52ERMYHHgNOALcBCEZmulFoRO0YpdUfC8bcC4xIuEVBKje2WzrjoJ32N\nRqNJROjOJ/2JwFqlVIVSKgw8D1ywi+OvBJ7rhnexU3rFoF9SkMa4Xy2n7KhTOGWWYKak8fqp8Njz\nK/jOY1dy4q/nYQVb+fe9J/G/M/4PnyGc/NIvufyJ+Tx8cjEzb3mGcFRx9l2n8JaM4M1X5wIw5rTJ\nXDcygyUPPsmc2jZ+cs5o7P88woJ/rWRb0GJ8bipHXHccbWPO5cmPNrL+s1W01VWRllfK2hmLWdwU\nImAr+qZ6GDaqkP6nToCRU/hsWytvLN9G9aZG/NUbKJo8juigcaxrCLFgYwPrNjTSVF1LsKEaK+gH\nwJeRQ1peKVn5mTRu9bMtaHXQ6NNMI260lpOfSmZxOhmluTQFLZoiUVrt6A5Ga4lma5keoS5mtBaw\nCIcsIsE27HAAOxSIJ0RFI+2JUVFvOsqXTtSbSiiWlBVVhO0oIddoLfYa0/ANo2NylunxYJpOUpaT\nnOUUUInarpbvJmjFErMS9XxwdHJTZKeFU2JJVYaboLUrEvV82HliVkzPT2RPk4Z29Q8r8Vr78g/w\nUDNaOygSs4i5bHbboF8GbE5Y3+Ju2/G+IgOAQcDshM2pIvKxiHwkIhfu5VvqgJZ3NBqNpgO7f4BI\noFBEPk5Yn6aUmraXN74CeFkplfh0MkApVSkig4HZIrJUKbVuL68P6EFfo9FoOuLKO0lSq5SasIv9\nlUD/hPV+7rauuAK4OXGDUqrSfa0QkXdx9P59GvR7hbyj0Wg0+5NulHcWAsNEZJCI+HAG9h1m4YjI\nSCAP+DBhW56IpLjLhcAUYEXnc/eUXjHoB0oGsPa911j68NnMe/YZpv/+Bp6adD0Xjyzg9Qnf4rNX\nn+PKW64i47E7mbGlma/dcizT/ENY9J9/sfa2G3hreytfOqoPObf/hnue/YT6isWUTzyNRy8+ksYn\nf8Jb720CYFx4NZ/8diYLG4KUpnqYcOZgci/+Ov/6vJb3P9xEw4ZlGB4f+UPHs3xVPduCFjlegyPy\n0xhwykjSJ5/NxkgGb6+uYd3aOho3rybYVIPvyBPYRjbztzTx4Zpa6rY5c/TjRmupjtFaRmEpuUXp\nVAYsmq3oDsXQ830mhSkeMoozyOybRWZZEfVhm1Y7uoPRmimOlp9qGGR6nNbWGiYciLhma2GsgH+H\nYugxPR9wzdbS2oumdDBaa2+BiN1urObxOc3rvrp6vmka8UIq8Xn6dkfjtUSTtcSWqOX7Omn7RsIc\nfVOSN1pTUXu3Rmv7Mke//Ro7n6Pf3Xp+b+Zg0fPB6YvpkaTa7lBKWcAtwCxgJfCiUmq5iDwgIucn\nHHoF8LxSSiVsGwV8LCKLgXeAXyTO+tlbtLyj0Wg0nehOp1Kl1ExgZqdt93da/1EX580Djui2jrjo\nQV+j0WgSEHc22KGKHvQ1Go2mE3sQyO116EFfo9FoOnEoD/q9IpC7YeM2vvfzO3h35CQmX3U1BT+/\ngQ1tEU78aBa3/OBvlE8+lycmWPzpodlcMCCHtPue4CePvI4vI4fnXlzBmJxUjn3ifm6f8TmrZr9O\ndr/h3HTFkQzfNJuPfvM2G9oiTClIo+KRX/Puku0AnDAsn2E3fJkV0pen317HtuWfYIcDZPcbzpAj\nS1ntD2EKDM/0MWjqAIpPPYXm4tG8t6Ge95dVU7N+C221VaioTaB4BEuqW/lgTQ3btzTTvHUDwaZa\nolYYw+MjJSuP9IIycoszGNgni1rXQM1WToJVmilkewyKUkwyStLJ6ptJZlkRGWVFNEW6CuI656S6\nAeBMj5CZ4iHYFiHkGq3Fgrh2KIAdDmJ3qlYFoLzpRMRDyIoSdKtmBSNR2iLtiVlBO0ogbLuJWDsa\nrcWCt6bHwDCdBC3bUk4AdydGa0CHfnRltBavpiXEE7NiP8m7CqomJmbtqrpVotFa5207Y0+CuD0Z\nsNxfFbP2YA5770Sc95hM643oJ32NRqNJQHAeTg5V9KCv0Wg0icihba2sB32NRqPpRG8uLr87esVv\nGG96FjetnMYbW5qZfRY88uSn3PPEV5jy208JNdXy+o9OY+Zx12GKcPrMR7nw9x9Su3ohJ1xxHn4r\nykXfO40308Yx/YX3UFGbo846nm8dlsniH/+et7a30j/Ny6TrjuaD55ZS5RqtjbnxRAITvsSjcyqo\n+PRzWms2k5ZXSv/DR/PVyQMI2Ir+aV5GHVHMgLMmwREn8/HWVl5bspWt6+vxV2/ACvoxPD7WNoSY\nW1HH6ooGGiq3dWm0ll2YQ1FJJkf2z93BaC3bY8aN1rL6ZJJRmktmWRGeojI3Mauj0VqseErMaC3H\na5KSnUI4YDmJWQlGa3Y4uIPRWoyY0VowwWgtlpAVM1oLhJ0W1/M7Ga0ZHiNutBYrpLIzo7XEPsRN\n3zolZ8WTtEwjruN7DcPR9jv9Q40lZu1Mz9+d0Zoh3avBH6xGaweag63rjuFacq030uPdFhFTRD4T\nkdfc9UEiMt8tKPCCm5qs0Wg0BwexyQFJtN7I/viuug0n/TjGQ8AjSqmhQANw/X7og0aj0SSJYJhG\nUq030qO9FpF+wDnAU+66ACcDL7uHPAN0i0e0RqPRdAein/T3id8CdwNRd70AaHRNiGDXBQVudIsH\nfFySEuSHt7/CDx+/kocn3shXjinj2ZHXsfjfz3PbvdcjD1zPa1tb+Mb9Z/CLrX1Z9J+XGXzCBbx4\n9TiumDoQz7ce4q6nFlJfsZjBU87ksUuPpObR+5j5zkZMgVOn9KP/Td9mYUOA/mlejvnSCLIvvYnn\nlm3ng7kbadiwDNOXRtHICZx+TDlnDc0n32cytjSTQWceQeqx57E2mMrMFdWsW11H46bPCTbVIIZJ\nWl4JH25u5KM1tdRWNbvF0OsBx2gtNa+ErOI+5JdkcHhZDiOKMncwWitKMSlK95JRnEFWvxyyyktI\nKS3FU1q+wxz9mJafYTomazleE1+6l5RsH6FAhHAggBXwEwn6XaO18A5GazGc+fmOyVrIUrSEdjRa\n8wct2sL2Lo3WTI/76mr8yRqtxYq0J2O0ZhgdDdd2ZrSWyO6M1mKbO8/bT2Rfjda6Q4vfn3p+d89N\nP9j0/BjdWSP3YKPHBn0RORfYrpT6ZG/OV0pNU0pNUEpNKCwo6ObeaTQaTdeI0HUyYBetN9KTUzan\nAOeLyNlAKpANPArkiojHfdrfVUEBjUajOSD01gE9GXrsSV8pda9Sqp9SaiCOV/RspdRXcHyhL3EP\nuwb4T0/1QaPRaPYUIbmn/N76xXAgkrO+CzwvIj8FPgP+fAD6oNFoNF0iAj5tw7BvKKXeBd51lyuA\niXtyfu2yVVw6fhyPDbkWHy8z7H9vcPZ5P+SIcy/jvrRPueeJhXzlmDLqr32Qh6//PZmlA3n6juOo\n/M7VTHjqt5z3j0Wsfe81CoaO54fXHkW/hX/npd/PoSpocV6/bMbeex3z7H74DOGk8aUMvfmbzPVn\n8fSsT9m69EPscIDC4UczZkIZV43vR2H1Ig7PTmHI6UMoOv1sanKH8uayauYt3Ubt+vW01TlGaylZ\n+WSWDOKtFdVs29hI89YKgk21TnDSl0ZqTiEZReXklWQyon8uR5TlMDQ/nZmu0VqmxyDPa1KUYpLV\nN5Psfk61rIy+xXhKyiGneIcgrs9oN1rL8Rqk+UxS81JJy0slFIi0V8uKhB2jtYgTzO0qkOtUyooS\nsnasltWakJgViNgdgrimxzFYixmtiUg8Scs0jR2M1qJWGGV3DOJCu+lah6QsjxEP3noTErQSDbAS\ng7h7Y7SW+AC3N0Zr8XO7MFrr7iBusvfvnut9QYK44pj8HapoGwaNRqNJQDi0NX096Gs0Gk0i0nv1\n+mQ4dIUrjUaj2QucJ30jqZbU9UTOFJFVrvXMPV3sv1ZEakRkkdu+nrDvGhFZ47ZruuP99YonfVvB\ngP+9wVnn3IN/wTSG3PkaGUX9mffdyTzRdyKjslKY9PqrHHHfbNrqqrj7gVsZ+8lf+OWfPyX78kzm\nvfwSvowcLvvyiVycvZ337vkLc+sCjMlJ5ZjvnkHN2Iu4/9lPubM4k3F3XMDm/lN46OWlrP/4U4JN\nNWT1GcLg8SP5+rEDGW7UUTv9RUYcU0b/807BGn0yc9Y0MP2TSrZWbMdfvQE7HMCTmklmyUAKy0uo\nWFdPU1UlbXVV2OEAYpj4MnLIKContyiDsr5ZHNk/h5GFGZRmOP9LEvX8nOIMsvtlkV1eTFZ5CWZJ\nOUZxOXZWyQ5Ga2luUlamxyDba5Lm6vmpealYAT92OOC+dl04JZGQpRzDNSuaoOdHCVlO4ZRYYlas\niIrh8bUXTYmZrZkS1/cNQzA9gm1FidpRbMuKF07pKjErRgfDNRG8RruOHzNaM2XHn+S70vOdWEFH\no7WdFU7prPPvKQeicMrBrucf7HTXk76ImMBjwGk4yagLRWS6UmpFp0NfUErd0uncfOCHwARAAZ+4\n5zbsS5/0k75Go9EkYEh7BvjuWhJMBNYqpSqUUmHgeeCCJLtyBvCmUqreHejfBM7cqzeVgB70NRqN\nphPODLHdN6AwZhfjths7XaoM2JywvjPrmYtFZImIvCwi/ffw3D2iV8g7Go1Gs7+QLqTCXVCrlJqw\nj7ecATynlAqJyDdwjChP3sdr7pRe8aRfethgJn/jacqOOoVTZgnVS+cw45GrmXvsaVQFI1z72gOc\n8dfPWf/BdCZdcRk/HObnxRv/TFPE5jePv02goZpx553FQ2cMZtnd9zJzZS2lqR5Ou+pIMq/9AT+b\nvY6V73/GUbeeAGffwm/f38CS91fSvGU1qTlF9DtyHF89aTBTyzMJz/4Ha/79KcMumowx8TwWbm3j\n34sq2bSqlqZNKwi11GN4fKQX9iW3XzmDh+RTV1mLv3oDkdYmwCmckl7Ql5ySQorLshk/II/RRZn0\ny/aRGap3C6E7en5BXiqZfTPJ6pdHVnkJvrIBePsOJJpZSMiXBSTq+UKG6czPz/EapOT4SHX1/NS8\nDKygn0ig3Witq8IpMcQwCdpOQXR/2MLvzsdvi9j4Q1a7nh+xCYQtDK8P0+NxLGdjc/I90mG+vmE6\nlrWJhVN2ZbQW0/vjhmsJ8/I7G63F5ul3VThlZyRTOGVPjdYSr9P5/P1ltNYbJp4c7CGCbszIrQT6\nJ6zvYD2jlKpTSoXc1aeAo5I9d2/oFYO+RqPR7C9iyVnJtCRYCAxzi0f5cCxppne8n/RJWD2f9voj\ns4DTRSRPRPKA091t+4SWdzQajSYBQbrNhkEpZYnILTiDtQk8rZRaLiIPAB8rpaYD/yci5wMWUA9c\n655bLyI/wfniAHhAKVW/r33Sg75Go9EksIea/m5RSs0EZnbadn/C8r3AvTs592ng6W7rDHrQ12g0\nmg4c6jYMvULTX1ETIdRSz9KHz2bes89w9wO3kvvgDby4dDvf/uk5/KJtDB/+458MPuEC/vfNibx7\n4U18VB/gylMHUfP5RwybegF/ueYoah+6nf/MWIOtFGefWM6g797Hk0vrmTlzOfUViym8/i6eXrSV\nmW+tpXb1QkxfGsWjJ3HeiYP40shCZN6LrH5hDkuW1ZBx8sWstbJ5eXEVS5dtp379CgIN1fFqWbn9\nh9N3UB5TRxXTUrW2Q7WstIK+ZJf2o6BPJuMH5HFEn2wG5aaSb4Tw1G8kx03KKkr3kt0vi5zyXLIH\n9iG1f3+8fQdiZ5diZxbREDgBOkAAACAASURBVHSCiV1Vy0rPTiE1103Myk0jJTeLSNBJzooZre0q\niCuG2WW1rNbwjkHceOUsM9FobSdJWh6jQ7WsxGByV0HcRMO1xGpZsQQtb2y7G9Dtiq4Ss3Z4z7uo\nlmUIHWzXdhfETbxmjJ0Fcfd2bNHVsnoQXURFo9FovjjE/PQPVfSgr9FoNJ3Qg75Go9F8QTAO8SIq\nveKdBZsbeeOp23l35CQmX3U1d9e/zG//9DHfvGgES790P7958BnyB4/hP9+fypqvXcyLS7dz4eA8\nxj/9R0rHTOW335hEyZuPMuPR96kKWpw9LJ9xP7md1/3F/PGlZVQvnYM3I4fXa1N5asZKqhbPQUVt\nCocfzXHHDeSao/qRv2EuG16cwbIPNrPaH6IyawjTV1Yzd1EV1WtW0VqzOV44JbvfCEoH5HHyYSVM\n7pdHoKE6XjglLa+ErJIBFJZlc8TAfMb0y2FEYTp90g08dRsIVyyn0GdSmupx9Px+2WQP6kNGeRne\nPgNReX2JZhXTEIrSGLTjhVPa9XyDjDRPB6O1tIIcUguysUMBolZkl4VTYnq+GGZcx28JtxdO8Qct\nx2wtZOEPRuKGazG93uM12w3WEgqnxLV+Q3ZaOKWznh/D5zHwGsZOC6eYRsciKrszWou/110UTknU\n83d2frLowintHPR6PmhNX6PRaL5ICHFfnUMSPehrNBpNJw5lK2k96Gs0Gk0CAjud/nso0CsG/X79\nSzFuvow3tjQz+yz43tinOX9oPvlPvsKZN/wZMQweu+9CMh67kydeWskx+Wmc+vKD/GhxlO9/6wRO\n2P4O/73jORY3BTm1OIPjHrqG5X1P4MdPLWDDgtmIYVJ+9FQemr6CDQvmEWltIn/wGEZPHsatxw9m\ncOsaKp9/jlUzVrOsOUTAVry+po4Z8zezdfVG/Ns2ELXCeDNyyC4bTunAIo4dXcxxA/MZUZBC1Apj\neHyk5hSSWTqI/D5ZDCvPZfyAXA4rzqQs04u3bi3W+mW0rl3j6PllWeQOzCF7UCnZA/vg6TsIKeyH\nlVVCk2XQELTZ2hKKm6zF9PycVE/cZC29MJ1UV89PLchx5ujvonBKop4vhok/bOMPW+1Ga0Fnjn5L\nyIrPzw+EbayI7Wj5icZqMQ0/Yc6+x/Ugj+n50d0UcYkXRk8wVzNE8JrSoXBK4nKyej7sqOd3Zb4G\nziBgiOyRnt/Vg2JnPb+75+gf7Hp+r8H9rB2q9IpBX6PRaPYXAniTLIXYG9GDvkaj0SSg5R2NRqP5\nIuFOCT5U0YO+RqPRJBCL4Ryq9ArhKrdpK0++toYfPn4lD0+8kVFZKZz06TtM/d4smqvW8f37ruW0\nxU/yp4dm0zfVy+V/+RbP2qP50xOvcUNhNe99/SFmVbcyPjeVU39yAdunfI3bnl/E6vfexQr4KR0z\nlavOHcnncz6kra6KrD5DGHbMkXz7lGGM8dRQ+9JfWPHiIhY2BGiKRClKMXn+w41s/rySxs0rsYJ+\nPKmZZPcZQsngMsaPLmbqsEIOL04nbfsqxDCdIG7JIArL8hk8IJdJg/MZU5JN/ywvqY2bsDetJLD2\ncxrXbCa/JIPcAdnkDCwhZ0gZ3n5DMUsHYef0oVVSaQjZVLWEqGwJkpYQxM3zOUlZ6YXppBekkVqQ\nFQ/ienLzsd1qWbEAalfEgrim10dLyKmY1eKarMWN1sJ2/DUctrGt6I7GarGELI9TLcuTUEy6c1LW\nzozWYq3dXM2IV8lKTNRqr5zV/j6SNVlLXI4FcTsEd/fto7vTf2DdH3Tt7ut1/6DXm8ZRx9hv9603\nop/0NRqNJgFxHygOVfSgr9FoNAkc6vKOHvQ1Go2mE71VukmGXvEbZuu2Fr575/E8NuRaAK5a/DIT\nfzqXLQv/x1V3fI3/s+fxhxv/hinCDQ9fwrvDL+f+R96kadNKPrzmTv69qo7hmT7Ou/sU7Ct/wC2v\nLGXpm3MINGyjePQUzj9rBDdP6kfL1nVkFPVn6DETuf3MEUwtsmj5959Z/vf5LKhqoSZkk+M1GJ+b\nyvplW2ncsIxIaxOmL43M0oEUDxnMEaOKOX1kMeP6ZJLTtJ7wsrmk5hSRWTKIgv7F9B+Qy7HDChnX\nJ5uBuT4yWqtRm1cSXL2MhtWbaVhTQ97gXHIGFZMztAxfv8F4+g7GzimlzZNJXcBmW0uYrS0htjQE\nyPYY5PtM8n2mY65WmEZ6YRpphVmkFuSQXpyHNy8PI7tgl3p+YlKW6fXFk7M6JGUFLfwhi5ZgJK7n\nWxEbKxLF8Bh4vB1N1zxeI15YJabnp3iMdsO13ej5MRL1fI9pJGj47Xq+12z3S0lGz49fW3av5xsi\ne6VHd3fhlJ3epxcMUL3pwVloN/DbXUvqeiJnisgqEVkrIvd0sf/bIrJCRJaIyNsiMiBhny0ii9w2\nvfO5e4N+0tdoNJpEurFGroiYwGPAacAWYKGITFdKrUg47DNgglKqTUS+BfwSuNzdF1BKje2Wzrj0\niid9jUaj2V84mn5yLQkmAmuVUhVKqTDwPHBB4gFKqXeUUm3u6kdAv258OzugB32NRqNJIGbDkEwD\nCkXk44R2Y6fLlQGbE9a3uNt2xvXA6wnrqe51PxKRC7vj/fUKeac4L5UPvvwgP/3Wg/gXTOO4Z7ex\nctbLnPGtG3h8xHaePPHnNERsbv/xWaw58y6+9cAbbF8xlyEnXcgLv7uNvqleLrppMjm3/4ZvvLKM\nD2e8h796A4XDj+b0c8Zw78mDSZn9FGl5pQyeNJmbzhnJuQNSCb7yW5b+dQ4frm2gKmiR6XH0/GGn\nDKShYjHBphoMj8/V84czcmQhZx5WwtF9syhsq8JaNpfajz4lo2gCeWWl9C3PZcqwQsb3yWFwbgrZ\nwVpkywqCa5dQ//lG6ldto2F9I0POHEHe8P6kDhiCt3w4Vk5fAil51LZZbPOHqWwOsqmhjY11bRzr\ndebop+c7Wn56YTppBZmkFeU5en5uLkZuMWZe0W4LoRseH2LGlr34w5ZbLKVdzw+EnSIqgaAV1/Ot\niO2YqiXMzzdMiev5aT4zruf7PGZS8/MhZrgW7VLP9yYsO0XRZY9M0VTU7lAIHXau5+8Nyer5+5wH\n0ANa+aE8cyUpBPZgxmatUmpCt9xW5CpgAnBiwuYBSqlKERkMzBaRpUqpdftynx570heRVBFZICKL\nRWS5iPzY3T5IROa7QY0XRMTXU33QaDSaPSU2ZbObArmVQP+E9X7uto73FDkV+D5wvlIqFNuulKp0\nXyuAd4Fxe/3GXHpS3gkBJyulxgBjgTNF5BjgIeARpdRQoAHn54xGo9EcJIhr5737lgQLgWHuw64P\nuALoMAtHRMYBf8IZ8LcnbM8TkRR3uRCYAiQGgPeKHhv0lYPfXfW6TQEnAy+7258BukWn0mg0mu6g\nO5/0lVIWcAswC1gJvKiUWi4iD4jI+e5hvwIygZc6Tc0cBXwsIouBd4BfdJr1s1f0qKbvTlf6BBiK\nM21pHdDo/iFgF0ENNyByI0Cf9NSe7KZGo9HEcWwYui+uoZSaCczstO3+hOVTd3LePOCIbuuIS4/O\n3lFK2e4c0344U5dG7sG505RSE5RSEzIGDeeb//cbyo46hVNmCZ+89A+mXHMt/znF5O8n38Zqf4ib\n7zyR+msf5MpfvEPVJ7MYcOx5TLt1Cvk+k8u/No4+9/2OO/+7ilmvzKF5y2ryB4/hpHMm8MPTh5H7\n4T/49JcvMeiY47jx3FFcMTIX67+Ps/TP7zBvWQ2bAxEyPQZjclIYedIABl80lUDDtoQg7khGjC7i\ngjF9ObZ/DiXhaqylc6j9cCFV8yvI79+fvgOdIO5RZTkMzU8lN9KAbFlBaPVn1C9bT8OqrTRUNFJT\nGyBveDmpA50grp3Tl1B6AXUBi+2tYTY3BdjUGGBjXRtb6tvI95lk5qWSXphGRkkGGcVZpBXlkVaQ\ng68gHzOvGDOnACMrn6gV3uHv3DmIa3p8GB4vhseHP2TR1BbpEMRtCVqEEpKyrIhN1IrGK2d5fCaG\nKfEErcSkLJ/HbK+clWQQF4hXzdpZENcbq57Vxad5ZxW5oD2IG6ugFf+buK+xJ7l9iWseyCDu3lz/\nCx/EdRFJrvVG9svsHaVUo4i8A0wGckXE4z7tdxnU0Gg0mgOJsc9fyQcvPTl7p0hEct3lNJyMtJU4\n2tQl7mHXAP/pqT5oNBrNniLoJ/29pQ/wjKvrGzgBjNdEZAXwvIj8FCf9+M892AeNRqPZY3qDn9He\n0mODvlJqCV3MKXXnm07ck2tVbNhG+ZensPThs8mZfBOTr7qaty7M5h8TrmRxU5D/u/042m5/lIt+\nOptNH75G+eRz+dMdxzFhxfOUXjuW/g9O485ZG3n1uXdp3LCM3IGHc+J5k3nwnFEUf/wCnz74d97+\nZCvf+PVorjmiEHvG71j02CzmLapmQ1uENFMYk5PCESeWM+zSqXiPvwTD8wsySwdSNHQ0w0YXceHY\nMqaU59InUkN02Rxq535E1fx1VC+rofSCXE4YUcTkAXmMKkynwGrAqFxBePVn1C1ZR93KSurWNLB9\neyvbghZpQ4bhGzgSO68/oYyieFLWpqYgmxoDVNS0srG2lebGIJl5qWQUZziavqvnpxfnkVJc6Oj5\necUYOYVE03J2+LvuSs83vL4Oer4/GInr+eGQhRWJErWcZkWipKZ7u9Tz03xmBz3fZxp7pOerqO0a\nrslu9fzOevSu9PwYiXq+ITvX8/fmJ7HW83spvfgpPhmSHvRF5FhgYOI5Sqlne6BPGo1Gc8AQkp6D\n3ytJatAXkb8BQ4BFQOxRSQF60NdoNIccWt5x/CBGK6VUT3ZGo9FoDgYO4TE/6UF/GVAKbO3Bvmg0\nGs0BR5dLdCgEVojIAhxPHQCUUufv/JTuw5OWycpHz+GdkZOYfOujvPOlTJ49ygni3nbnCbTd8Xsu\neOBtNs6bQfnkc/nznScwcfk/mf71J7hw3Txud4O49RWLyR88hhPPm8yvzh/tBHF/9gxvLaiiKmhx\n95FFRGf8js9+P5P3P9sWD+KOz03lyJMHMuzyU/CecBmfWznxIO7oI0q4cGwZJwzIpa9VQ3Tpu9S8\nP4/KeWupXlbDqpYwU0cV7xDEDa1Y0GUQtzZsx4O4wYwiatwg7oaGwA5B3LbmEBnFGR2SsnYWxO0c\nyN1VENdMScP0+HYbxI0laNlWNOkgrs9j7FEQF0g6iJuow+ogrmZfOITH/KQH/R/1ZCc0Go3mYOJQ\nLjSS1KCvlHpPREqAo91NCxLd4DQajeZQQbqxXOLBSFJfaCJyGbAAuBS4DJgvIpfs+iyNRqPpneiM\nXMfc/+jY072IFAFv0W6R3KMc3i+L1wdNYE5tG7PPgqeOuorV/jDfue90ar7+EF+6/w0qF85k8AkX\n8Lc7T+CwD5/g5ZufZW5dgNdnVPDfF96madNKCoaO54wvHcvPzhpB/ty/svBn/+TtRdVsC1oMTPcS\neflXfPbYG7y/tN1kbXxuKkecOpChl5+GecLlrAxm8OLiKkqGHcbhR5bwpXFlHNc/h9LQVqzF71Dz\nwXy2zFvLthW1rPVHqA5ZnDcwnxGFaRSE65xKWSsWULtkHXUrqqhfW09NbYDKgEVDxMZvRbHyBxBK\nL6CmzaKy2TFZW1/vVMpK1PPbWkId9PyMPgXtJmsFpUh2IdHULEfTT8mK/z2T0fNjhmtd6fkxk7WY\nnm/b0aT1/BSPsUd6vlPhKjk9P6bFJ6Pnw64rZXXW82Uv/4XvTs/v7gfKXjoOHVQIWt4BMDrJOXUc\n2n8XjUbzBWZvv+R7A8kO+v8TkVnAc+765XTyh9ZoNJpDAtHJWSil7hKRi3HKdQFMU0q92nPd0mg0\nmgOD4NRwOFRJ2ntHKfUK8EoP9mWn1C9dxUdGH374+JU8PPFGmi2bex+5mM/OuJvr7vk31cvmMOqM\nS3jhjuMo/c8v+Pt3/8WnjUGmFqXzrWdfw1+9geLRU7jwoqN54PShpP7vD3z081eYvbKWmpDNkAwf\nJ0wuY+GvZ/LBmnqqghY5XoOj89I47OwhDLr8XIzJF7G4yeS5zzbz/qIqxo/vw4Vu0ZSi1k1EPptN\n9fsLqPpoPVWf17HWH6Y6ZBGwFaOL0skNVMOmpQRWfhrX8+vWNrC9PsC2oB3X88NRRVtqPrWtFpXN\nITY1BVlf1xrX8/2NQVqbQwT8IUItzWT2yUnQ8wswcosx84ocPT8tB5WWg52SSVvE0coT9XzT63OX\nvZi+NAyvD9Pjc5Y9PhrbwgTCNoGg1aFoihW2idoK252rb8eKqLhafmLRlDSfic+MrTttT/R8oIOe\n7zXb9fvOer5pJK/nQ9d6fndp+YnXj6H1/N7DoSzv7FKXF5EP3NcWEWlOaC0i0rx/uqjRaDT7Dycj\nN7mW1PVEzhSRVSKyVkTu6WJ/ioi84O6fLyIDE/bd625fJSJndMf72+WTvlLqOPc1a1fHaTQazaFE\ndz3nu/VEHsMpIrUFWCgi0zsVOL8eaFBKDRWRK4CHgMtFZDRwBXAY0Bd4S0SGK6W6/umaJMnO0/9b\nMts0Go2m9+PIhcm0JJgIrFVKVSilwsDzwAWdjrkAeMZdfhk4RRx96QLgeaVUSCm1HljLHtYi6Ypk\np10elrgiIh7gqH29uUaj0Rx0JJmY5Y75hSLycUK7sdPVyoDNCetb3G1dHuPWDm8CCpI8d4/Zpbwj\nIvcC3wPSEjR8AcLAtH29ebKEo4ofv3YPv/GehI+X+d6Lt/Fc3wu55+6/0bxlNUdd+hVevfkY7N9+\nmyd/9Q7rWsOc1y+bkx/7Olf/eCllR5/NdZcewd3HlRP820+Y89DrvLWpCb8V5fDsFKZMHcCob17C\nzy58iJqQTVGKyaT8dEZeNIr+l1yAmnghH1S18fynG1mwqIrqNRU8cPmlTCzLIrduNaFP3mLrnI+p\n/GgTWyoaWesPUxu2CUcVpkCefzPR9YtpW76IuuUV1K6opqGikW2NQbYF25OybNe4urrNCeJuaAyw\nsa6Niho/VfWBeBA32BYm1NJMpK2J9NEFZJQW4C2ImawVQWZB3GTN9qbTGo7SFom2J2QZJqbXScBK\nrJTlcQO4sSQtf9AiHLa7DOJaYRvbdpKzonYUnxvA9XkM0n1mh6SsxCCuz2N0COK2B23bg7iJgddo\n1E46iNvVk9fOgrgxkg3i7mnQVQdxey+iFLKbz00CtUqpCT3Zn+5ml0/6SqkHXT3/V0qpbLdlKaUK\nlFL37qc+ajQazX5FVDSplgSVQP+E9X7uti6PcVWUHJwE2GTO3WN2N3tnpLv4koiM79z29eYajUZz\n8KFARZNru2chMExEBomIDycwO73TMdOBa9zlS4DZbsGq6cAV7uyeQcAwHA+0fWJ38/S/DdwI/KaL\nfQo4eV87oNFoNAcd3VQkUCllicgtwCzABJ5WSi0XkQeAj5VS04E/A38TkbVAPc4XA+5xLwIrAAu4\neV9n7sDup2ze6L5O3dcb7Qt9DhvEV6rGMPNPj+JfMI271hTy57ufIGqFOfMb1/LCFaOouPVK/vnc\ncvxWlC9P7Mvk393NwpLjGXriPO7+yli+XK7Y/svb+fDxD5hT2wbAlII0jr5wBENuuJbGUadTE/o5\n/dO8TByQzciLx9Ln4ktpHX4isysaef7jzSxbUs32dZ/j37aBEwfkkLr5E1o/epPKOYuoWlDF+i3N\nbA5Y1Cfo+TleE/vz+bQsW0zdsvXUraqloaKRSn+YmpCTlBWw2/V8nyFU1AfY1BRkQ20rFTV+qhsC\ntDaHaGsK0eYPEWltItzWhBXwk1lWhLewBMMtmkJGLtHUHKKp2UTMFNrCNq2RKG0RtVM9P9FkzUxx\ndH2Pz0soZGGFY8VSbDcZyymgkqjn25aVoOUbu9TzfYmGawl6fueErGiCpuo1DQxht3p+Z0k/GT1/\ndwZr+6q9d3W61vMPcpRK9ik+ycupmXSyrVFK3Z+wHMRxMO7q3J8BP+u2zpD8lM1LRSTLXf6BiPxL\nRMZ1Z0c0Go3mYKEbNf2DjmSnbN6nlGoRkeOAU3F+jjzRc93SaDSaA4WCqJVc64UkO+jHfiefg2O2\n9l/A1zNd0mg0mgOIojsDuQcdyRquVYrIn3BSiR8SkRT2o5/+yjqbJb//E+WTz+WUWcJH//wDmaUD\nueP2i7hnsJ95p53NSwuqyPeZfO3SUYz+5UP8syaPX/xuHo/fPJnjqWD1vT/nnZc/Z3FTkByvwfGF\nGYz52kTKrvsGFdmj+OvcjYzKSmHCkcWMuGwieed9ha05w/nf8u08v2AzG1Zsp75iGW11VUStML4V\nb1M/dzaVH6xg6yfbWFvbRlXQoiliYytHm8/xGvRN9VI/fz51yzZQv7aBuo1NVAacAuhNEUf7T9Tz\nMz0Gq+taqdjeysa6VuoaArQ1h5z5+a1Bwi31RIJ+rIAfOxzEWzwEs6AUM684XixFpeUQxENbOEpr\nJErAitISsuKGaolz8x39Pq2jnu+ap0VCdnw+vqPpq3gBlZimr6I2USsc1/PTfJ4OBVNiOr5pyA6a\nPuxez1e23UHP7zxXH9r1fCNB3d6dnh87D5LT8/fGf6un5+Z3dQ9Nd6Ag2jsH9GRIduC+DCf6fIZS\nqhHIB+7qsV5pNBrNAeRQ1vST9dNvE5F1wBmu09v7Sqk3erZrGo1Gc4DopQN6MiQ7e+c24B9Asdv+\nLiK39mTHNBqN5oCgFETt5FovJFlN/3pgklKqFUBEHgI+BH7fUx3TaDSaA0VvlW6SIdlBX2ifwYO7\nvN9iSIGmBqbc8VXeuHUyOZNvonzyuUz79vFM/vxFXj3mcd7a3sqYnFQu+M5U8r7zCHfNWstLL79F\n9bI5HHN2LfN//lfe/LCSqqBF31QPJx1ZzJgbTyb9/BuZ25LBtFmrWDB/Cy+cNpDhV5yC98TL+NzO\n55VPKnl94RYqV1fRtGklgYZtAKRk5VP92nQqP1zDtsXbWdXiVMnyW84HJc0UCn0eytI89ClIY9v8\nNdStaWD79ta4wVpTxKmSBU5ptlgQN9tjsryymY21rTQ3BmlrDtHWEiLU6ifS2tQhiGuFAnhKyjFy\nCuMGa9GULNosRVvEptWKEohEaQpaNIWsHYK48QCuWzXL40vBMA08PhOP18RKMFuz3eBth8QsK+wE\nciNhJ4DrJmR1DuLGm2lgiOyySlYsiKvshOQsw9hlQlYsgCuSXAA38X67C+LubQGlZIK4+1KdSQdw\ne5LuTc462Eh20P8LMF9EYnVxL8SZq6/RaDSHHl/0QV8p9bCIvAsc5266Tin1WY/1SqPRaA4U3WzD\ncLCxOz/9VOCbwFBgKfC4a/Kv0Wg0hyTCF1vTfwaIAO8DZwGjgNt7ulOd6duvlHdOj/DmyEkce9vv\n+PcNR9Pwo2/w8OMfURWM8KVh+Zz4h5upOPJSvvzH+SyZ9R7+6g3klI/irese4Z1tfgJ2lPG5qUw+\nczDDb7iC8DGX8s+VtTz97lLWfbqOho3LGP2rG7HHncPsjc28+Nk6Pl60leo1a2iuWocV9GN4fKTm\nFJLdbwRrZjzO5g2NrG+NdCiYkukxKElx9Pzisizyh+VTtaCKquYQ24I2zVbHgimmQJppkOkxyPOa\n5PsMZlY1428KEmgJE/CHCLU0xg3W7HAQKxwgGgkTtcJIfh/sNNdgzZNGWzhKwFK0RqK0hm2aQhZN\nwQj+sI3pS91Rz08wWDNNA4/XxPAYeLwG4ZC1U4O1mJYfS85K85k7NVgzDcFnGngNwTBklwVToKOe\nr6L2bvX8uC6fpNCdqOfvqmBKouS+L5mIh5qevw9d7yUosHvnzJxk2N2gP1opdQSAiPyZPfByFpH+\nwLNACU5i8zSl1KMikg+8AAwENgCXKaUa9rzrGo1G0wPEbBgOUXb3ABOJLeyFrGMBdyqlRgPHADe7\n1d3vAd5WSg0D3nbXNRqN5qDhi5yRO6ZTbdxYrVwBlFIqe2cnKqW2Alvd5RYRWYlT1PcC4CT3sGeA\nd4Hv7u0b0Gg0mu7lCxzIVUqZ3XETERkIjAPmAyXuFwLANhz5p6tzbsSp2kVZTiYPTb6Z2rDF22co\n3jv2JF5Ztp2SFA+3fm0sQ376G57e6OHhn81m04I3ASiffC6XnjOSGec+Rr7P5NTyPI687hhKrvoG\na9IGM+3Ndbw5dyNVyxbhr96AitpsHX4GMxdt48WPNrHp8xrqK5bQVleFitp4UjPJKO5PfvkwSgfm\nsuzVejYHInF93mcI+T6TkhQP5Vk+8gbnUjCikLzh/Xl/VkXcYC1gt1fkaZ+bb5Dj6vk5WSk01rQS\n8IfjBmvhtibsUAAr2IrtavkxPdzOKkKlZBFQJoEEg7XGgIU/7MzP94csmkNWgn7ftcGax2vi8Rlx\nbb+1ObTD3PyYhh/T8+OavtfcQc+Pmax5DQNTnGIoXkN2a7AWX3a3ew1jh+LniXq+IcnpzJ3n8Cdj\nsHYwafl7fv/uvdehr+UncAgP+j3ulCkimcArwO1KqebEfW4dyC7rkimlpimlJiilJhRkpPV0NzUa\njcbhELdh6NFBX0S8OAP+P5RS/3I3V4tIH3d/H2B7T/ZBo9Fo9gyFsiJJtX1BRPJF5E0RWeO+5nVx\nzFgR+VBElovIEhG5PGHfX0VkvYgsctvYZO7bY4O+OL9j/wysVEo9nLArsfL7NcB/eqoPGo1Gs8co\n9teTfjKTWtqAq5VShwFnAr8VkdyE/Xcppca6bVEyN03WhmFvmAJ8FVgqIrHOfA/4BfCiiFwPbMTx\n6tdoNJqDAoXqEFvqQXY7qUUptTphuUpEtgNFQOPe3rTHBn2l1AfsPI/klD25VlVVE0W5+dz820t5\neOKNrGsNc16/bE7+/XVUTvk6Z72wmEWzPqB5y2qy+gxh9NRj+cH5h3FKdhNPZKcwZeoARn3zEtRJ\nV/PSqjr+9O9FrFu0kbq1nxJpbcKTmklOv+H84p11fPRZFdtWr6N56zoirU2IYZJe0JesPkMpHljK\n0CH5nDSymJX+cDwhpAf3dAAAH8FJREFUK8drxA3WSvtkkj8sj/zhfckbNYCUQSPZHJix04SsbI9B\nvs+kMMVDemEaGSUZtDQECLU0E2lrItzatENCVmJA0krLpzUSpS1eIcumJWzRFLTwh22aQhH8QYum\ntgje1EwnOcsN4naVkBUL6Joew62WtfOErHggN2rHK2ftLCErFsz1mEZSCVmJ7C4hq3PVrK7oyoht\nTxKy9jQAeyCDuN0dwIUvWhCXPamcVSgiHyesT1NKTUvy3KQmtcQQkYk4ZWrXJWz+mYjcj/tLQSkV\n2t1Ne/JJX6PRaHohak+km1ql1ISd7RSRt4DSLnZ9v8MdlVL/396Zh8dxl3n+81Z1t9SSbN2SZcu2\nHN8mISGHQ8jAhCSQwJJjsyEkMAyzS8bDcj/AkIQsEOaZeTYwswnLwgLmZiYDA4E8BAiYJORYjhCc\nxE7s2I4dH/FtWZd1tNRdXb/9o37dqpa7pZYPSe1+P89TT1f9qrqqfnbr7erve4lI3qAWe5424F+B\n9xiTDS26g+DLIgasJfiV8A8T3bAafUVRlDDGnLSTdvRU5spC+0TksIi0GWMOjhfUIiKzgV8Cdxpj\nngqdO/MrYUREvgN8oph7mrLm5oqiKKWByUqXEy0nyYRBLSISAx4Avm+MuX/MvkwUpBCUu99UzEVL\n4km/ua6S/7rll3xhc5oY9/PJj76O9s/cwz0b+vnGp3/D/mcexonEOOsN1/Hua1fygYvbqXzye2z4\n8v284663Un/TGl6UuXz1F9t48g+vcPDF5xjs3AtATWsHjYvP4axXtfCrX2+ld/cLJHoOY/w00epa\nZrV2UNfeQduiei5d3sylixp4VUs1G31D3BXqoy5zKiPMr62gYWkDDUsaqV+5kJolS4h2rMRvXEhf\nalQfjLtC3B3V8htiLrNqK6hpqaaqKU5N22yGug7nFFgbm5AVpmc4ndXzM81SBqymP5gMtPzeoRQD\nIx5uLJ6TkBWJuYGmH0rICmv7WU0/1CwlnJDlhz788ZCm71oNP+oGRdJGdX3J6s0TJWSFt6NuSMPP\nk5AV1vjHMtEf5qnW8vMx3jmKLRJXLJqQdQrIRO+cfvIGtYjIhcD7jDG32rE3AI0i8jf2fX9jI3Xu\nE5FmAt/pBoKKyBNSEkZfURRl6jCTceSe+FWM6SJPUIsxZj1wq13/N+DfCrz/8hO5rhp9RVGUMIap\nCtmcFtToK4qi5DCp6J2SoySMvte+iPPv2cqOJx5i4Om1POKu4u33PMtLTzxCcrCP5hWv5dI3ncPn\n3rKCpV3Psuv2/8EzP9rEU90JPnHfg9z7/EHuf+Jp9mx8kb59L5FOJqisbaau42zmr5jHla+Zy9tW\ntvKG7/076WQCNxanqnEus9uXM6ejnvOWNfH6xY2cP3c2HbOjRA9vGy2uVhWhcWEtTcsbqVvWTt3y\nRUQ7ViBti/Hq2ulJB//EMUeIu0K1O6rl11dHiTdVUdNSRXVrNfGWeqrnNJD41aFs4/NCWr44LuK4\n9A6PxuVn9Pz+kUDLHxgO1geGU/QPe0Sra0fj8LMN0B2r44/R9yMOXjIVXD+dG5dv/DTpzLZ9Ispo\n+hkNP2qboEfdQMePOoJrNf1iYvPD22OLq40dg+O18WKcbGP1/PG0/BPV3gvp+arlz2BOYfTOTKQk\njL6iKMrUoU/6iqIo5cPURe9MC2r0FUVRQhhMto/zmYgafUVRlDD6pD/9vLz7ENHHfk77RW/minXC\n8+u+xsDh3dQuWMmFN1zLXdes4tKKwxz+2t/z62/+kd8fGaQ7mWZOZYR3fvvP7Nywm+5dG0kN9hGt\nrqW+42zmrjiLS8+byzVnz+HCtmpmH3kR46ezDtyWhS0sW9zAXy5v4eL2WhbXVxDv2Y2/fiPHNm3g\n3NoKWubNChKybHG1WMcK3HnLSNe3c8yponPQY9+xIWoiQXG1etsdqz4Wobq1iqqmKqpbqqhqqaW6\nrZF4cz3R5laSP9mRt7haBnFcnEgMcVwODozQbztj9SdHHbi9iRQDwymGkmkGhj2SyTSxikhOAlY2\nOSvq4kYEx3WIhZKs0iOJvMXVsg7ddCg5K+rmLa6W6ZjliGTXi3XgZnBDna3CxdVyHLuMOjOLzZQs\nJiGr3By4UOZOXAgcuankdN/FaaMkjL6iKMrUMTXJWdOFGn1FUZSxqLyjKIpSJhhzKoqpzVhKwuhH\nKqu57R8/yu1/2UHtJe9nVttiVt/8bu68bhVX1g3Q/f3P8eg3f8/vXumjcyRNc4XL29pmseKGlfzz\nAz/LNkppXHI+c5Yt5qJz27j2nDYuaZ9FXfd2RtY9zK4n19O84mqaFrSybGkjl61oYfW8OhbXx6jp\n34//3HMMbn2eo8/v4OiLh1n++vk5jVLc9kDL73Nr6Ex47D82yO7eBHu6hphbGTmuUUpGyw8SshqJ\nNjbh1rfg1jfjJTZMqOW70aAZysH+kaDAWiJF31CQhDUw4tE/nMpq+V4qjZfyicWjxzVKiUSd47T8\niohDPBYhnUxMqOVn7rMi4oyr5WcStdwCunuhPzLjpyfU8mG0wcpk/1iL1fJPVuZWLb+00OgdRVGU\ncsEYTFqNvqIoSllgjMFPedN9G6cNNfqKoihhDPqkP92cPX82H97+LR7/u9/wuo98ic9cs4o3xI9y\n5Dt38cg3/8D/OzhAdzLQ8q9pn83KG8+m/cbr8c+/BnP5bTQtu4i2ZYu4xMblXzS3htqjWxn59SPs\nemI9B/68jz0v9/D6ez6ejctfVFdBdd8r+M89x8CWQMvv2tZJ9/ZuDhwb4aYv3kzF4lXZuPxep4rO\nIY99xwZ5pS/Bzs5B9nQNsu/oEB+fVXGclp+Ny29swm2cg1vfAtX1+PHavMXVxmr5TiSKE4nxSs9Q\nUFht2KMvkcyJy0+NBHp+Ou3jJdNUVkfHjcsPmpvbbdc5rlFKPi0/o31Wuk7BuHxHglj7TIPz8PzG\n0/IzZPwA42n5MPk2cJnjp1PLP5Hznw49X8lFjb6iKEqZYIzB13r6iqIo5cOZHL2jjdEVRVHC2Oid\nYpaTQUQaRORhEdluX+sLHJcWkQ12eTA0vkhE/iQiO0TkP2wT9QlRo68oihIiE71TzHKS3A48aoxZ\nCjxqt/ORMMacZ5drQ+OfB+41xiwBeoD3FnPRkpB3up/fymc+3EPMER69yrDrix/gJ7YzViJt6KiK\ncuXyRlbcdAGtN7yD3vmr+enOHn5430Zec9312c5Y5zRXEt31JwZ+9AjbntjIgWcOsfNAP3sTKbqT\naT5z1fJsZ6zU75+hZ/MmujbvomtrFz07e9k7lKJzxOOY5xO//O2k69rpTEfoHPLY09vP3r4Eu6wD\n91DXEIPHRhg8NsKc81pyOmPFW+qJ1Dfj1LcQaZyDX1WHXzELP15L0gm+rDOdscRxcaIxHOvMzThw\n3Yo4biTGvp5EtjPWwLAXJGIlfZuQZR25nsH3fKpnV+Z0xorHXCqsEzfswM2MeclEtjhaPgfu6HpQ\ncC3TGWu0S1auAzdw7o5fFC1vUlqBwmpjHbiFipwVopADN99ZJptcdTocuMrU4U+NI/c64DK7/j3g\nceC2Yt4owYf3cuCdofffBXx1ovfqk76iKEoYG7JZpLzTJCLrQ8uaSVyp1Rhz0K4fAloLHFdpz/2U\niFxvxxqBXmNM5ufGPmBeMRctiSd9RVGUKWNyGblHjTEXFtopIo8Ac/LsujP3ksaIiClwmoXGmP0i\nchbwWxF5Aegr9gbHokZfURQlhOHURe8YY64stE9EDotImzHmoIi0AUcKnGO/fd0pIo8DrwF+AtSJ\nSMQ+7bcD+4u5p5Iw+knf8Pbz2zhvzRu5Z/UaXh5MEneFc2srOff181l+yxuJXnYz200j39l8iIce\nfIq9W/fT98oWNvzoDtr9LvxNP+fod37P/j9u59DGI+wYSHJg2GPAC/5z466w5NBTJB9/hgMv7ODo\npr10be+hq3OQ/QmPnlSavpRP0g++jPdWLuBwV4rdvQPs7h5iZ+cg+7qH6OsdZvDYMIn+JIn+flKD\nfbRdvISqlnoqmhqyiVhObRN+vBavchZ+ZS1DnmEo6ZPwPKvdxxDXxQ3p+E40RiQWDzT9WBwnGmPP\n0UFGQkXVvNB62vNJp318+1pZHSWWo+WP6viZQmux0OKnkjm6feYPITwG4PtpKiM2Ictq+VHHydHx\nw7p+scXWMrgZ7X4CLf9kdfexbz/VRdJUxy8RjMFPTkkZhgeB9wB329efjT3ARvQMGWNGRKQJuBT4\ngv1l8BhwI/DDQu/Ph2r6iqIoYQz4vl/UcpLcDbxJRLYDV9ptRORCEfmmPWYlsF5ENgKPAXcbY160\n+24DPiYiOwg0/m8Vc9GSeNJXFEWZKgxTU2XTGNMFXJFnfD1wq13/A3BOgffvBFZP9rpq9BVFUcIY\ncvo4n2mUhNFvW7WQRese5isbDhDjfm6+oI2VN11I8w3v4kjzOfzk5R5+8MArvLx5I0df3sxg5158\nL4kbi1P3439i2+9e4OCzh9hxcJADw0FMftpAzBGaK1xaKyIsqIqy/Ytf5ujWLnp297E/4WVj8hNp\nn7T1q8ccIe4Kv9x+lJ1Hgpj8oz0JBnqHGRpIMjyYJNnfTXKoDy8xQDo5TOPFF2QbpPhVdfiVtaQr\nZ5F0YgymfIYGPRIpQ99Iiv6RNNF4TTYm362wGn5Ix3dj8WwjlGN9I3lj8jOF1oxvSHsevpeksSaW\njcmPR90cHd91JEfPjzpBwTU4PiYfAh0/g0mnqYg4eWPyw9vhRijhc41H0ETl+KJq+XT8E6lDVmxM\n/mRzACa6hjKTMVqG4UQQkW+LyBER2RQaKyrtWFEUZdqYXJx+yXE6HbnfBa4eM1Zs2rGiKMq0YIwh\nnfSKWkqR02b0jTFPAt1jhq8jSBfGvl6PoijKjMJYSXPipRSZak2/2LRjbDrzGoAFbQUPUxRFObVo\n56zTwwRpxxhj1gJrAarnLTMXr/kWffteYuDptfTOX83DO3v44eN72bb5UTp3vMjgkb2kkwncWJzq\n5vnMbl9O64I6fvypD2YLqqVNkOhTG804byM0LqylaXkjdcva+dm/PFbQeVsTEapdh4aYS0PM5et/\n2JMtqJbPeeuNJPC9ILkp+qq/wq+sJWULqg2mfIaGfRKpFP1Jj75hj74Rj4GkR/+IR6ymPltQLZ/z\n1nUdIjGXSNRhoC+Rdd6m00FClp/2s85bk05n76OhpiKnoNrYxbXF0jKdr3wvFfxfFHDeZtfDyVkF\nnLfhomkTOXDH7s8kZ43nvD2Rn6xhB+uZ5LzVxloniQGTLmiaSp6pNvpFpR0riqJMFwYzVVU2p4Wp\nzsjNpB3DJNKGFUVRpgwDxjdFLaXIaXvSF5EfENSKbhKRfcBnCdKMfyQi7wX2ADedrusriqKcCMZA\nOqnJWZPGGHNLgV3HpR1PRKK3h1h/N/MuuIIr1gl7tjxE7+4XGOo6EGjm1bXMmruYhgWLmdNRxyXL\nmrn0rEbOaanmf3562CZhRZhbGWFeTYyGpfU0LGmkYeVCapYuIdaxAr+pg42f/lX2mnFXiLsOsyMO\ntVGX5gqXWbUVVDXGqWmtZvfmA6QG+3J0/HQqmdXPw7p0f/1iBlOGRMJnKJXM0fAHRjyOjXj0DdlG\nKCMe8fo52aSsSNQlEsvo+LYBStTFiThEog5HXukb1fLttTOF0owf6Pm+XW+ZVTGq39tkrKjjEHUl\nq+c7jn21hdHG0/HDVEXdnIJoYR1/VHeXgnrzeDq/iIw2UQm93xlzzGQ5ruDaOOc41cXXnFMsvKuO\nfwoxRjV9RVGUcsJXo68oilImaMimoihK+WAAv0SdtMWgRl9RFCWMMerInW7mzGvlp9/4COe2VlF7\nyftxY3Hi9a3MveAqWhfU8eplTbx+SRMXzZtNx+wo0cPbSL30C/p/vYmrWqtpXFhLw5J6GlYuoG75\nIqIdK5C2xaTr2ulJR+gc8tjTM0xt1MlJwKqvjhJvqqKmpYrq1mriLfVUNddR1dZIzzc25iRgjXVE\niuNml61dw/QNB47bvpEgAatvKMXAcLA+MGyduMMeXipNTVNTTgKWE0rKciMSOHdtB6w9m/flJGBl\nlnRmOz1aHbN5dsVxCVhRN3DaRp1M16vR9XQqmZ3PRN2uoo6Tk4AVrqiZM17g/ePhWo/teI7bE3W0\nFnLequO2fDGanKUoilJGqNFXFEUpJzQjV1EUpXyYoozcYvqLiMgbRWRDaBkWkevtvu+KyK7QvvOK\nuW5JPOm3JDqp+MjN/PaZQ7zuY/8nJ/mqLTKMe3ALya2P0f3zrWzfupejW7voOjDA/oTHmn//cDb5\nyqubR+eQR+egx+6eIfbsGu1+1dMzzKc76rLJV1UtNVS11FM1p4F4cwNOfQuRxjk4dc348VqG/+Xz\nOfcY1vCdaNDpyolEcSIx/vBKT07yVUbDH7Iavpfy8ZLpbLer2saqbPJVpshaJqmqIuIQj0WCbdfh\nqf7ubPJVRsMPd7kKluCppaEympN85Y5Zd4RA83dHk7My5NPgw2MRNzf5ypFR/T6ctFXoXOPhkKu9\nH5dUNamzhd43zjmPO3aS5z7VGn4Y1fNPL4Ypi9PP9Be5W0Rut9u35dyLMY8B50HwJQHsAH4TOuTv\njTH3T+aiJWH0FUVRpgxj8Kcmeuc6glI1EPQXeZwxRn8MNwK/MsYMncxFVd5RFEUJYUzwpF/McpIU\n3V/EcjPwgzFj/yQiz4vIvSJSUcxF9UlfURRlDJPoitUkIutD22ttLxAAROQRYE6e992Zc70J+ovY\nUvTnAOtCw3cQfFnECHqP3Ab8w0Q3XBJGf/++Xr6+7yXirvDoVYaRLQ/R/cNtdG3Zx8tbu+g8NMCh\n4TRHkx4Dnk8i9A286dXvZFdvgj3bhtjZuY09Rwfp6x1m8Ngwif4kw4ND2cJpF374CuKtzbj1zbj1\nLVn93o/X4lfMYsAXBlOGoZSPE4nl1e+daIxILCiW5kRiuBVxHttypKB+7yWD5ifhJigrz59LLOJQ\nFXOJRdysfp/R9MONT5KDfcDx+n1Y14egAUp9PJqj30cdJ9vsJF/zk4k0/TAxJ9PgJFe/z/yUzNcA\npVjc0JvGvv1k4ukLvVcl8zLHTOop/qgx5sLCpzJXFtonIpPpL3IT8IAxJhU6d+ZXwoiIfAf4RDE3\nrPKOoihKGBunX8xykkymv8gtjJF27BcFEjxRXQ9sKuaiJfGkryiKMlUYpqzgWt7+IiJyIfA+Y8yt\ndrsDmA88Meb994lIM8GP0w3A+4q5qBp9RVGUMMaQTp5+o2+M6SJPfxFjzHrg1tD2bmBenuMuP5Hr\nqtFXFEUJYQz4RsswTCvNsyv45H97HQ0rO7hn9Rp6UmkGPJ+kzYhzJXAk1kQc5lZGaYg5NFdEqGqI\nc+v//SND/SOMDA4EDtvBPrzhQXwviTeSyHaXApj1V3eTrpzNQMpnMOWT8HwSKZ++bo++kX4GRmzH\nqxGPWXMXWwduDDcWtw7cilBXKzdbHO2VnT2kraPWS6YxxmQ7XY3tcmX8NGfPW5nT3Sq7hIqkRR0H\nV8AbHgRyHbaQv8tVfTya12E7tjBaMUlUxxdcy5wj12FbqNPVZBDyO11PpFvW2PMWixZMKy/SavQV\nRVHKAwOcwfXW1OgriqKMRZ/0FUVRygTfkJWOz0RKwuj7C87iqb/+Aru6h4hxP4urozTEXGY1VlHV\nFKe6tZrqlllUzWmkqqWeWGMDbmMbbn0z2279aVazD5MtjmYTqNxIjPt3JekbORRo97ZAWl8iRSLp\n0T/skQglWM1Zviqr2Wcanjiu3bYNTjLJVE+uez5Hs89XIC2cTLWibVZWs4+4wWug5Y+uZ4qlZfwS\nY8k3NrsikqPZZwqkjW1wktGvJ1MYLeJKwSYnJ9uQxB1zglPd4CQ4p2r2yigq7yiKopQJBqPyjqIo\nSrmgjlxFUZQyQ43+NLN9z2H+9kP/Cz+VZODptVBdHxRBq5xNKhJnKOWT8Ay9KZ/9yTR9Ix59wykG\nkmni9a3HFUJzK4LXSCwa6PE2tv7eB1+0xdByC6D5aZ+05wV6vI2rv+qa87Ox82OLoGVj7F2HqCM8\n9P1dOYXQwlp5vrj6pQ3V4xZCCzcrSScTRf0bGj9NTSxQ3ccWQYP8cfWTIVaE7n6icfXhhiynksno\n+KrRlw/GaPSOoihK2WDQ6B1FUZSyQTV9RVGUMkPlHUVRlDIh0PSn+y5OHyVh9N1YJS2rLsWNOFyx\nTvCS3XipTpsolSbtGXzPz3ajMr4h7Xn4XpI3v/M/WeeqSzzq5nSfGlvQ7NOf/W4oScrP230qwy0X\nXIsjHOdozed4He47mn1fMQlPC2qDVpfFdJ+aTAJVdTQ4Uz6f5MkmPEXd3BOcSr+ne5q8qOqcVQqh\nT/qKoihlggGmpIXKNKFGX1EUJYTBaPSOoihKuRBE76jRn1bOXtjA77/0NgBqL3n/pN773a+9vehj\nP9a5t+hjL50/q+hj8xV8G4+W6tPz31IVPdE2JhMTOR1V0CyqvStTyhnuyD19VmAcRORqEdkmIjtE\n5PbpuAdFUZR8ZJ70i1lOBhF5u4hsFhHfNkMvdFxeeykii0TkT3b8P0QkVsx1p9zoi4gLfAV4C7AK\nuEVEVk31fSiKohQibYpbTpJNwA3Ak4UOmMBefh641xizBOgB3lvMRafjSX81sMMYs9MYkwR+CFw3\nDfehKIpyHD5BGYZilpPBGLPFGLNtgsPy2ksJ4rcvB+63x30PuL6Y64qZYoeFiNwIXG2MudVuvxu4\n2BjzwTHHrQHW2M2zCb4VzxSagKMTHlU6nGnzgTNvTuU0n4XGmOYTPbGI/NqevxgqgeHQ9lpjzNpJ\nXu9x4BPGmPV59uW1l8BdwFP2KR8RmQ/8yhhz9kTXm7GOXPsPtxZARNYbYwpqXqWGzmfmc6bNSedT\nPMaYq0/VuUTkEWBOnl13GmN+dqquMxmmw+jvB+aHttvtmKIoyhmFMebKkzxFIXvZBdSJSMQY4zEJ\nOzodmv6fgaXW8xwDbgYenIb7UBRFmenktZcm0OUfA260x70HKOqXw5Qbffut9EFgHbAF+JExZvME\nb5uURlYC6HxmPmfanHQ+MwwR+c8isg+4BPiliKyz43NF5CGY0F7eBnxMRHYAjcC3irruVDtyFUVR\nlOljWpKzFEVRlOlBjb6iKEoZMaONfqmWaxCRb4vIERHZFBprEJGHRWS7fa234yIiX7JzfF5Ezp++\nO8+PiMwXkcdE5EWbNv4RO16ScxKRShF5WkQ22vl8zo7nTWsXkQq7vcPu75jO+y+EiLgi8pyI/MJu\nl/p8dovICyKyQUTW27GS/MzNJGas0S/xcg3fBcbG+t4OPGqMWQo8archmN9Su6wBvjpF9zgZPODj\nxphVwGuBD9j/i1Kd0whwuTHmXOA84GoReS2F09rfC/TY8XvtcTORjxA4+zKU+nwA3miMOS8Uk1+q\nn7mZgzFmRi4EHu11oe07gDum+74mcf8dwKbQ9jagza63Advs+teBW/IdN1MXgtCwN50JcwKqgGcJ\nshyPAhE7nv38EUROXGLXI/Y4me57HzOPdgIjeDnwC4LmZSU7H3tvu4GmMWMl/5mb7mXGPukD84Bw\nreN9dqxUaTXGHLTrh4BWu15S87RSwGuAP1HCc7JSyAbgCPAw8DLQa4IQOci95+x87P4+ghC5mcQX\ngU8y2vSpkdKeDwQFL38jIs/YsixQwp+5mcKMLcNwJmOMMSJScrGyIlID/AT4qDHmmIQK3ZfanIwx\naeA8EakDHgBWTPMtnTAi8jbgiDHmGRG5bLrv5xTyF8aY/SLSAjwsIlvDO0vtMzdTmMlP+mdauYbD\nItIGYF+P2PGSmKeIRAkM/n3GmJ/a4ZKeE4Axppcgs/ESbFq73RW+5+x87P5agjT4mcKlwLUispug\nCuPlwP+mdOcDgDFmv309QvDFvJoz4DM33cxko3+mlWt4kCBVGnJTph8E/tpGH7wW6Av9fJ0RSPBI\n/y1gizHmntCukpyTiDTbJ3xEJE7gn9hC4bT28DxvBH5rrHA8EzDG3GGMaTfGdBD8nfzWGPMuSnQ+\nACJSLSKzMuvAmwkq7ZbkZ25GMd1OhfEW4K3ASwR6653TfT+TuO8fAAeBFIG2+F4CzfRRYDvwCNBg\njxWCKKWXgReAC6f7/vPM5y8I9NXngQ12eWupzgl4NfCcnc8m4DN2/CzgaWAH8GOgwo5X2u0ddv9Z\n0z2HceZ2GfCLUp+PvfeNdtmc+fsv1c/cTFq0DIOiKEoZMZPlHUVRFOUUo0ZfURSljFCjryiKUkao\n0VcURSkj1OgriqKUEWr0lWlHRNK2kuJmW/ny4yJywp9NEflUaL1DQtVOFaXcUaOvzAQSJqik+CqC\nRKm3AJ89ifN9auJDFKU8UaOvzChMkHK/Bvigza50ReSfReTPtk763wGIyGUi8qSI/FKCngtfExFH\nRO4G4vaXw332tK6IfMP+kviNzcJVlLJEjb4y4zDG7ARcoIUgm7nPGHMRcBHwtyKyyB66GvgQQb+F\nxcANxpjbGf3l8C573FLgK/aXRC/wX6ZuNooys1Cjr8x03kxQU2UDQTnnRgIjDvC0MWanCSpm/oCg\nXEQ+dhljNtj1Zwh6HShKWaKllZUZh4icBaQJKigK8CFjzLoxx1xGUA8oTKGaIiOh9TSg8o5StuiT\nvjKjEJFm4GvAl01QGGod8N9taWdEZJmtugiw2lZhdYB3AL+z46nM8Yqi5KJP+spMIG7lmyhBP95/\nBTIlnL9JIMc8a0s8dwLX231/Br4MLCEoI/yAHV8LPC8izwJ3TsUEFKVU0CqbSkli5Z1PGGPeNt33\noiilhMo7iqIoZYQ+6SuKopQR+qSvKIpSRqjRVxRFKSPU6CuKopQRavQVRVHKCDX6iqIoZcT/B0qy\nR2TEdZliAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 画出位置编码的图\n",
    "def plot_position_embedding(position_embedding):\n",
    "    plt.pcolormesh(position_embedding[0], cmap='RdBu')\n",
    "    plt.xlabel('Depth')\n",
    "    plt.xlim((0, 512))\n",
    "    plt.ylabel('Position')\n",
    "    plt.colorbar()\n",
    "    plt.show()\n",
    "    \n",
    "plot_position_embedding(position_embedding)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "QRTZuyovW4OM"
   },
   "source": [
    "### mask构建\n",
    "1. padding mask\n",
    "2. look ahead"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RBgy8ZABW4ON"
   },
   "outputs": [],
   "source": [
    "# 1. padding mask\n",
    "# batch_data.shape: [batch_size, seq_len]\n",
    "def create_padding_mask(batch_data):\n",
    "    padding_mask = tf.cast(tf.math.equal(batch_data, 0), tf.float32)\n",
    "    # [batch_size, 1, 1, seq_len]\n",
    "    return padding_mask[:, tf.newaxis, tf.newaxis, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 161
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 272989,
     "status": "ok",
     "timestamp": 1581685060625,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "dyszNH_cW4OV",
    "outputId": "4d24f743-f44a-40b2-d83a-341673463ce3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 1, 1, 5), dtype=float32, numpy=\n",
       "array([[[[0., 0., 1., 1., 0.]]],\n",
       "\n",
       "\n",
       "       [[[0., 0., 0., 1., 1.]]],\n",
       "\n",
       "\n",
       "       [[[1., 1., 1., 0., 0.]]]], dtype=float32)>"
      ]
     },
     "execution_count": 21,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test\n",
    "x = tf.constant([[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]])\n",
    "create_padding_mask(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ocfW9mMPW4Oc"
   },
   "outputs": [],
   "source": [
    "# 2. look ahead只能看到前面的，看不到后面的\n",
    "# attention_weights.shape: [3, 3]\n",
    "# [[1,0,0]\n",
    "#  [4,5,0]\n",
    "#  [7,8,9]]\n",
    "# 下三角都是0，上三角都是1\n",
    "def create_look_ahead_mask(size):\n",
    "    mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n",
    "    # shape: [seq_len, seq_len]\n",
    "    return mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 89
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 272149,
     "status": "ok",
     "timestamp": 1581685060627,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "Qh_GkSAoW4Oi",
    "outputId": "2c8d8b83-9c53-47c5-aba6-97e3647f653e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 3), dtype=float32, numpy=\n",
       "array([[0., 1., 1.],\n",
       "       [0., 0., 1.],\n",
       "       [0., 0., 0.]], dtype=float32)>"
      ]
     },
     "execution_count": 23,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test \n",
    "create_look_ahead_mask(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "VEj4xC9kW4Op"
   },
   "source": [
    "### 缩放点积注意力机制"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ai_OTgTcW4Or"
   },
   "outputs": [],
   "source": [
    "def scaled_dot_product_attention(q, k, v, mask):\n",
    "    \"\"\"\n",
    "    Args:\n",
    "    - q: shape == (..., seq_len_q, depth)\n",
    "    - k: shape == (..., seq_len_k, depth)\n",
    "    - v: shape == (..., seq_len_v, depth_v)\n",
    "    - seq_len_k == seq_len_v\n",
    "    - mask: shape == (..., seq_len_q, seq_len_k) 默认为None\n",
    "    Returns:\n",
    "    - output: weighted sum\n",
    "    - attention_weights: weights of attention\n",
    "    \"\"\"\n",
    "    \n",
    "    # matmul_qk.hape: (..., seq_len_q, seq_len_k)\n",
    "    matmul_qk = tf.matmul(q, k, transpose_b=True)\n",
    "    dk = tf.cast(tf.shape(k)[-1], tf.float32)\n",
    "    scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n",
    "    \n",
    "    if mask is not None:\n",
    "        # 目的是使得在softmax后，值趋近于0\n",
    "        scaled_attention_logits += (mask * -1e9)\n",
    "    \n",
    "    # attention_weights.shape: [..., seq_len_q, seq_len_k]\n",
    "    attention_weights = tf.nn.softmax(scaled_attention_logits, axis = -1)\n",
    "    \n",
    "    # 加权求和\n",
    "    # output.hape: (..., seq_len_q, depth_v)\n",
    "    output = tf.matmul(attention_weights, v)\n",
    "    return output, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5d9GXsVWW4Ox"
   },
   "outputs": [],
   "source": [
    "# 打印缩放点积，方便调试\n",
    "def print_scaled_dot_product_attention(q, k, v):\n",
    "    temp_out, temp_att = scaled_dot_product_attention(q, k, v, None)\n",
    "    print('Attention weights are:')\n",
    "    print(temp_att)\n",
    "    print('Output is:')\n",
    "    print(temp_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 89
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 271295,
     "status": "ok",
     "timestamp": 1581685061876,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "xwAx1VeCW4O6",
    "outputId": "dac13d7c-6b0a-481a-b2e9-8ec536abb8d5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attention weights are:\n",
      "tf.Tensor([[0. 1. 0. 0.]], shape=(1, 4), dtype=float32)\n",
      "Output is:\n",
      "tf.Tensor([[10.  0.]], shape=(1, 2), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# 小数四舍五入\n",
    "np.set_printoptions(suppress=True)\n",
    "# test\n",
    "temp_k = tf.constant([[10,0,0],\n",
    "                      [0,10,0],\n",
    "                      [0,0,10],\n",
    "                      [0,0,10]], dtype=tf.float32)  # (4, 3)\n",
    "\n",
    "temp_v = tf.constant([[   1,0],\n",
    "                      [  10,0],\n",
    "                      [ 100,5],\n",
    "                      [1000,6]], dtype=tf.float32)  # (4, 2)\n",
    "\n",
    "temp_q1 = tf.constant([[0, 10, 0]], dtype=tf.float32)  # (1, 3)\n",
    "print_scaled_dot_product_attention(temp_q1, temp_k, temp_v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 89
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 271117,
     "status": "ok",
     "timestamp": 1581685061878,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "oI4wl3mTW4PA",
    "outputId": "376a4e1e-a771-4dbe-9a62-3eabce210727"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attention weights are:\n",
      "tf.Tensor([[0.  0.  0.5 0.5]], shape=(1, 4), dtype=float32)\n",
      "Output is:\n",
      "tf.Tensor([[550.    5.5]], shape=(1, 2), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "temp_q2 = tf.constant([[0, 0, 10]], dtype=tf.float32)  # (1, 3)\n",
    "print_scaled_dot_product_attention(temp_q2, temp_k, temp_v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 89
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 270966,
     "status": "ok",
     "timestamp": 1581685061879,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "x0zlMU8sW4PF",
    "outputId": "c69c9edb-f035-4fc4-ef27-5d680aa7d10d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attention weights are:\n",
      "tf.Tensor([[0.5 0.5 0.  0. ]], shape=(1, 4), dtype=float32)\n",
      "Output is:\n",
      "tf.Tensor([[5.5 0. ]], shape=(1, 2), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "temp_q3 = tf.constant([[10, 10, 0]], dtype=tf.float32)  # (1, 3)\n",
    "print_scaled_dot_product_attention(temp_q3, temp_k, temp_v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 197
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 270804,
     "status": "ok",
     "timestamp": 1581685061880,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "AaOYwTGJW4QG",
    "outputId": "c48c898d-d344-40b0-84c9-0c915e1ec0bf"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attention weights are:\n",
      "tf.Tensor(\n",
      "[[0.  0.  0.5 0.5]\n",
      " [0.  1.  0.  0. ]\n",
      " [0.5 0.5 0.  0. ]], shape=(3, 4), dtype=float32)\n",
      "Output is:\n",
      "tf.Tensor(\n",
      "[[550.    5.5]\n",
      " [ 10.    0. ]\n",
      " [  5.5   0. ]], shape=(3, 2), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "temp_q4 = tf.constant([[0, 0, 10], [0, 10, 0], [10, 10, 0]], dtype=tf.float32)  # (3, 3)\n",
    "print_scaled_dot_product_attention(temp_q4, temp_k, temp_v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "wXs6zWUrW4QJ"
   },
   "source": [
    "## 模型构建\n",
    "### 多头注意力"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jdWfS2CjW4QL"
   },
   "outputs": [],
   "source": [
    "class MultiHeadAttention(keras.layers.Layer):\n",
    "    \"\"\"\n",
    "    理论上：\n",
    "    x -> Wq0 -> q0\n",
    "    x -> Wk0 -> k0\n",
    "    x -> Wv0 -> v0\n",
    "    实际上：把x分成q,k,v\n",
    "    q -> Wq0 -> q0\n",
    "    k -> Wk0 -> k0\n",
    "    v -> Wv0 -> v0\n",
    "    实战中的技巧：\n",
    "    q -> Wq -> Q -> split -> q0,q1,q2...\n",
    "    \"\"\"\n",
    "    def __init__(self, d_model, num_heads):\n",
    "        super(MultiHeadAttention, self).__init__()\n",
    "        self.num_heads = num_heads\n",
    "        self.d_model = d_model\n",
    "        assert self.d_model % self.num_heads == 0\n",
    "        self.depth = self.d_model // self.num_heads\n",
    "        \n",
    "        self.WQ = keras.layers.Dense(self.d_model)\n",
    "        self.WK = keras.layers.Dense(self.d_model)\n",
    "        self.WV = keras.layers.Dense(self.d_model)\n",
    "        \n",
    "        self.dense = keras.layers.Dense(self.d_model)\n",
    "        \n",
    "    def split_heads(self, x, batch_size):\n",
    "        # x.shape: (batch_size, seq_len, d_model)\n",
    "        # d_model = num_heads * depth\n",
    "        # x -> (batch_size, num_heads, seq_len, depth)\n",
    "        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n",
    "        return tf.transpose(x, perm=[0, 2, 1, 3])\n",
    "    \n",
    "    def call(self, q, k, v, mask):\n",
    "        batch_size = tf.shape(q)[0]\n",
    "        \n",
    "        q = self.WQ(q) # q.shape: (batch_size, seq_len_q, d_model)\n",
    "        k = self.WK(k) # k.shape: (batch_size, seq_len_k, d_model)\n",
    "        v = self.WV(v) # v.shape: (batch_size, seq_len_v, d_model)\n",
    "        \n",
    "        q = self.split_heads(q, batch_size) # q.shape: (batch_size, num_heads, seq_len_q, depth)\n",
    "        k = self.split_heads(k, batch_size) # k.shape: (batch_size, num_heads, seq_len_k, depth)\n",
    "        v = self.split_heads(v, batch_size) # v.shape: (batch_size, num_heads, seq_len_v, depth)\n",
    "        \n",
    "        # scaled_attention_outputs.shape: (batch_size, num_heads, seq_len_q, depth)\n",
    "        # attention_weights.shape: (batch_size, num_heads, seq_len_q, seq_len_k)\n",
    "        scaled_attention_outputs, attention_weights = scaled_dot_product_attention(q, k, v, mask)\n",
    "        \n",
    "        # scaled_attention_outputs.shape: (batch_size, seq_len_q, num_heads, depth)\n",
    "        scaled_attention_outputs = tf.transpose(scaled_attention_outputs, perm=[0, 2, 1, 3])\n",
    "        \n",
    "        # concat_attention.shape: (batch_size, seq_len_q, d_model)\n",
    "        concat_attention = tf.reshape(scaled_attention_outputs, (batch_size, -1, self.d_model))\n",
    "        \n",
    "        # output.shape: (batch_size, seq_len_q, d_model)\n",
    "        output = self.dense(concat_attention)\n",
    "        return output, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 270190,
     "status": "ok",
     "timestamp": 1581685061883,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "bZPXtJfwW4QP",
    "outputId": "55aa5a02-2be0-4d16-fef7-8504f2369876"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(TensorShape([1, 60, 512]), TensorShape([1, 8, 60, 60]))"
      ]
     },
     "execution_count": 31,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test\n",
    "temp_mha = MultiHeadAttention(d_model=512, num_heads=8)\n",
    "y = tf.random.uniform((1, 60, 512))  # (batch_size, seq_len_q, d_model)\n",
    "out, attn = temp_mha(y, y, y, mask=None)\n",
    "out.shape, attn.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "48pcT5B-W4QS"
   },
   "source": [
    "### feed_forward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "h2nC7rjVW4QT"
   },
   "outputs": [],
   "source": [
    "def feed_forward_network(d_model, dff):\n",
    "    # dff: dim of feed forward network.\n",
    "    return keras.Sequential([\n",
    "        keras.layers.Dense(dff, activation='relu'),\n",
    "        keras.layers.Dense(d_model)\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 269206,
     "status": "ok",
     "timestamp": 1581685061885,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "FvhAxbamW4QW",
    "outputId": "f8db4804-94d4-470d-bc4c-f1e675e589d3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([64, 50, 512])"
      ]
     },
     "execution_count": 33,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test\n",
    "sample_ffn = feed_forward_network(512, 2048)\n",
    "sample_ffn(tf.random.uniform((64, 50, 512))).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0DAw6bqrW4Qa"
   },
   "source": [
    "### Encoder Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "atzZbs73W4Qb"
   },
   "outputs": [],
   "source": [
    "class EncoderLayer(keras.layers.Layer):\n",
    "    \"\"\"\n",
    "    x -> self attention -> add & normalize & dropout -> feed_forward -> add & normalize & dropout\n",
    "    \"\"\"\n",
    "    def __init__(self, d_model, num_heads, dff, rate = 0.1):\n",
    "        super(EncoderLayer, self).__init__()\n",
    "        self.mha = MultiHeadAttention(d_model, num_heads)\n",
    "        self.ffn = feed_forward_network(d_model, dff)\n",
    "        self.layer_norm1 = keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layer_norm2 = keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.dropout1 = keras.layers.Dropout(rate)\n",
    "        self.dropout2 = keras.layers.Dropout(rate)\n",
    "        \n",
    "    def call(self, x, training, encoder_padding_mask):\n",
    "        # x.shape: (batch_size, seq_len, dim=d_model)\n",
    "        # attn_output.shape: (batch_size, seq_len, d_model)\n",
    "        # out1.shape: (batch_size, seq_len, d_model)\n",
    "        attn_output, _ = self.mha(x, x, x, encoder_padding_mask) \n",
    "        attn_output = self.dropout1(attn_output, training=training)\n",
    "        out1 = self.layer_norm1(x + attn_output)\n",
    "        \n",
    "        # ffn_output.shape: (batch_size, seq_len, d_model)\n",
    "        # out2.shape: (batch_size, seq_len, d_model)\n",
    "        ffn_output = self.ffn(out1)\n",
    "        ffn_output = self.dropout2(ffn_output, training=training)\n",
    "        out2 = self.layer_norm2(out1 + ffn_output)\n",
    "        return out2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 267478,
     "status": "ok",
     "timestamp": 1581685061888,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "V-9p9oWEW4Qd",
    "outputId": "f70738b6-86b5-4821-e4bf-49473533f528"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([64, 50, 512])"
      ]
     },
     "execution_count": 35,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test\n",
    "sample_encoder_layer = EncoderLayer(512, 8, 2048)\n",
    "sample_input = tf.random.uniform((64, 50, 512))\n",
    "sample_output = sample_encoder_layer(sample_input, False, None)\n",
    "\n",
    "sample_output.shape  # (batch_size, input_seq_len, d_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "IdOB7p4NW4Qg"
   },
   "source": [
    "### Decoder Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "y1fG_7UIW4Qh"
   },
   "outputs": [],
   "source": [
    "class DecoderLayer(keras.layers.Layer):\n",
    "    \"\"\"\n",
    "    x -> self attention -> add & normalize & dropout -> out1\n",
    "    out1, encoding_outputs -> attention -> add & normalize & dropout -> out2\n",
    "    out2 -> feed_forward -> add & normalize & dropout -> out3\n",
    "    \"\"\"\n",
    "    def __init__(self, d_model, num_heads, dff, rate = 0.1):\n",
    "        super(DecoderLayer, self).__init__()\n",
    "        self.mha1 = MultiHeadAttention(d_model, num_heads)\n",
    "        self.mha2 = MultiHeadAttention(d_model, num_heads)\n",
    "        \n",
    "        self.ffn = feed_forward_network(d_model, dff)\n",
    "        \n",
    "        self.layer_norm1 = keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layer_norm2 = keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layer_norm3 = keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        \n",
    "        self.dropout1 = keras.layers.Dropout(rate)\n",
    "        self.dropout2 = keras.layers.Dropout(rate)\n",
    "        self.dropout3 = keras.layers.Dropout(rate)\n",
    "        \n",
    "    def call(self, x, encoding_outputs, training, decoder_mask, encoder_decoder_padding_mask):\n",
    "        # decoder_mask是由look_ahead_mask和decoder_padding_mask做与操作合并而来\n",
    "        # x.shape: (batch_size, target_seq_len, d_model)\n",
    "        # encoding_outputs.shape: (batch_size, input_seq_len, d_model)\n",
    "        # attn1,out1.shape: (batch_size, target_seq_len, d_model)\n",
    "        attn1, attn_weights1 = self.mha1(x, x, x, decoder_mask) \n",
    "        attn1 = self.dropout1(attn1, training = training)\n",
    "        out1 = self.layer_norm1(x + attn1)\n",
    "        \n",
    "        # attn2,out2.shape: (batch_size, target_seq_len, d_model)\n",
    "        attn2, attn_weights2 = self.mha2(\n",
    "            out1, encoding_outputs, encoding_outputs, encoder_decoder_padding_mask) \n",
    "        attn2 = self.dropout2(attn2, training = training)\n",
    "        out2 = self.layer_norm2(out1 + attn2)\n",
    "        \n",
    "        # ffn_output, out3.shape: (batch_size, target_seq_len, d_model)\n",
    "        ffn_output = self.ffn(out2)\n",
    "        ffn_output = self.dropout3(ffn_output, training = training)\n",
    "        out3 = self.layer_norm3(out2 + ffn_output)\n",
    "        return out3, attn_weights1, attn_weights2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 264547,
     "status": "ok",
     "timestamp": 1581685061890,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "VKroAdkMW4Qk",
    "outputId": "a7a88afa-aa6c-4c92-b7fb-feccc8fecd2d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 60, 512)\n",
      "(64, 8, 60, 60)\n",
      "(64, 8, 60, 50)\n"
     ]
    }
   ],
   "source": [
    "# test \n",
    "sample_decoder_layer = DecoderLayer(512, 8, 2048)\n",
    "sample_decoder_input = tf.random.uniform((64, 60, 512))\n",
    "sample_decoder_output, sample_decoder_aw1, sample_decoder_aw2 = sample_decoder_layer(\n",
    "    sample_decoder_input, sample_output, False, None, None)\n",
    "\n",
    "print(sample_decoder_output.shape)\n",
    "print(sample_decoder_aw1.shape)\n",
    "print(sample_decoder_aw2.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ZBkHX89SW4Qn"
   },
   "source": [
    "### Encoder Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NsABpvmbW4Qn"
   },
   "outputs": [],
   "source": [
    "class EncoderModel(keras.layers.Layer):\n",
    "    def __init__(self, num_layers, input_vocab_size, max_length,\n",
    "                 d_model, num_heads, dff, rate = 0.1):\n",
    "        super(EncoderModel, self).__init__()\n",
    "        self.d_model = d_model\n",
    "        self.num_layers = num_layers\n",
    "        self.max_length = max_length\n",
    "        \n",
    "        self.embedding = keras.layers.Embedding(input_vocab_size, self.d_model)\n",
    "        # position_embedding.shape: (1, max_length, d_model)\n",
    "        self.position_embedding = get_position_embedding(self.max_length, self.d_model)\n",
    "        self.dropout = keras.layers.Dropout(rate)\n",
    "        self.encoder_layers = [\n",
    "            EncoderLayer(d_model, num_heads, dff, rate) for _ in range(self.num_layers)]\n",
    "        \n",
    "    def call(self, x, training, encoder_padding_mask):\n",
    "        # x.shape: (batch_size, input_seq_len)\n",
    "        input_seq_len = tf.shape(x)[1]\n",
    "        tf.debugging.assert_less_equal(\n",
    "            input_seq_len, self.max_length,\n",
    "            message='input_seq_len should be less or equal to self.max_length')\n",
    "        \n",
    "        # x.shape: (batch_size, input_seq_len, d_model)\n",
    "        x = self.embedding(x)\n",
    "        # 做缩放，范围是0-d_model，目的是在与position_embedding做完加法后，x起的作用更大\n",
    "        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
    "        x += self.position_embedding[:, :input_seq_len, :]\n",
    "        x = self.dropout(x, training = training)\n",
    "        \n",
    "        # x.shape: (batch_size, input_seq_len, d_model)\n",
    "        for i in range(self.num_layers):\n",
    "            x = self.encoder_layers[i](x, training, encoder_padding_mask)\n",
    "            \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 263668,
     "status": "ok",
     "timestamp": 1581685062636,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "vKDmfC5CW4Qp",
    "outputId": "4cd3d4b6-c654-4f96-c7f9-d096b2d59bfa"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 37, 512)\n"
     ]
    }
   ],
   "source": [
    "# test\n",
    "sample_encoder_model = EncoderModel(2, 8500, max_length, 512, 8, 2048)\n",
    "sample_encoder_model_input = tf.random.uniform((64, 37))\n",
    "sample_encoder_model_output = sample_encoder_model(\n",
    "    sample_encoder_model_input, training=False, encoder_padding_mask=None)\n",
    "\n",
    "print(sample_encoder_model_output.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xKK4u4X4W4Qu"
   },
   "source": [
    "### Decoder Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gDebC3C0W4Qu"
   },
   "outputs": [],
   "source": [
    "class DecoderModel(keras.layers.Layer):\n",
    "    def __init__(self, num_layers, target_vocab_size, max_length,\n",
    "                 d_model, num_heads, dff, rate = 0.1):\n",
    "        super(DecoderModel, self).__init__()\n",
    "        self.d_model = d_model\n",
    "        self.num_layers = num_layers\n",
    "        self.max_length = max_length\n",
    "        \n",
    "        self.embedding = keras.layers.Embedding(target_vocab_size, self.d_model)\n",
    "        # position_embedding.shape: (1, max_length, d_model)\n",
    "        self.position_embedding = get_position_embedding(self.max_length, self.d_model)\n",
    "        self.dropout = keras.layers.Dropout(rate)\n",
    "        self.decoder_layers = [\n",
    "            DecoderLayer(d_model, num_heads, dff, rate) for _ in range(self.num_layers)]\n",
    "        \n",
    "    def call(self, x, encoding_outputs, training, decoder_mask, encoder_decoder_padding_mask):\n",
    "        # x.shape: (batch_size, output_seq_len)\n",
    "        output_seq_len = tf.shape(x)[1]\n",
    "        tf.debugging.assert_less_equal(\n",
    "            output_seq_len, self.max_length,\n",
    "            message='output_seq_len should be less or equal to self.max_length')\n",
    "        \n",
    "        attention_weights = {}\n",
    "        \n",
    "        # x.shape: (batch_size, output_seq_len, d_model)\n",
    "        x = self.embedding(x)\n",
    "        # 做缩放，范围是0-d_model，目的是在与position_embedding做完加法后，x起的作用更大\n",
    "        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
    "        x += self.position_embedding[:, :output_seq_len, :]\n",
    "        x = self.dropout(x, training = training)\n",
    "        \n",
    "        # x.shape: (batch_size, output_seq_len, d_model)\n",
    "        for i in range(self.num_layers):\n",
    "            x, att1, att2 = self.decoder_layers[i](\n",
    "                x, encoding_outputs, training, decoder_mask, encoder_decoder_padding_mask)\n",
    "            attention_weights['decoder_layer{}_att1'.format(i+1)] = att1\n",
    "            attention_weights['decoder_layer{}_att2'.format(i+1)] = att2\n",
    "        \n",
    "        return x, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 107
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 263140,
     "status": "ok",
     "timestamp": 1581685062639,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "ItdkiK_-W4Qx",
    "outputId": "175a50f1-9c03-4bd9-f6a6-1809a49b0ed1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 35, 512)\n",
      "(64, 8, 35, 35)\n",
      "(64, 8, 35, 37)\n",
      "(64, 8, 35, 35)\n",
      "(64, 8, 35, 37)\n"
     ]
    }
   ],
   "source": [
    "# test\n",
    "sample_decoder_model = DecoderModel(2, 8000, max_length, 512, 8, 2048)\n",
    "sample_decoder_model_input = tf.random.uniform((64, 35))\n",
    "sample_decoder_model_output, sample_decoder_model_att = sample_decoder_model(\n",
    "    sample_decoder_model_input, sample_encoder_model_output,\n",
    "    training = False, decoder_mask = None, encoder_decoder_padding_mask = None)\n",
    "\n",
    "print(sample_decoder_model_output.shape)\n",
    "for key in sample_decoder_model_att:\n",
    "    print(sample_decoder_model_att[key].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5mGhosRJW4Qz"
   },
   "source": [
    "### Transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ANg9_PnxW4Q0"
   },
   "outputs": [],
   "source": [
    "class Transformer(keras.Model):\n",
    "    def __init__(self, num_layers, input_vocab_size, target_vocab_size, max_length,\n",
    "                 d_model, num_heads, dff, rate = 0.1):\n",
    "        super(Transformer, self).__init__()\n",
    "        self.encoder_model = EncoderModel(num_layers, input_vocab_size, max_length,\n",
    "                                          d_model, num_heads, dff, rate)\n",
    "        self.decoder_model = DecoderModel(num_layers, target_vocab_size, max_length,\n",
    "                                          d_model, num_heads, dff, rate)\n",
    "        self.final_layer = keras.layers.Dense(target_vocab_size)\n",
    "        \n",
    "    def call(self, inp, tar, training, encoder_padding_mask,\n",
    "             decoder_mask, encoder_decoder_padding_mask):\n",
    "        # encoding_outputs.shape: (batch_size, input_seq_len, d_model)\n",
    "        encoding_outputs = self.encoder_model(inp, training, encoder_padding_mask)\n",
    "        \n",
    "        # decoding_outputs.shape: (batch_size, output_seq_len, d_model)\n",
    "        decoding_outputs, attention_weights = self.decoder_model(\n",
    "            tar, encoding_outputs, training, decoder_mask, encoder_decoder_padding_mask)\n",
    "        \n",
    "        # decoding_outputs.shape: (batch_size, output_seq_len, target_vocab_size)\n",
    "        predictions = self.final_layer(decoding_outputs)\n",
    "        \n",
    "        return predictions, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 107
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 260913,
     "status": "ok",
     "timestamp": 1581685062641,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "0V0BjgvQW4Q2",
    "outputId": "44c48726-64be-4cd4-abe3-94a77edea60e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 31, 8000)\n",
      "decoder_layer1_att1 (64, 8, 31, 31)\n",
      "decoder_layer1_att2 (64, 8, 31, 26)\n",
      "decoder_layer2_att1 (64, 8, 31, 31)\n",
      "decoder_layer2_att2 (64, 8, 31, 26)\n"
     ]
    }
   ],
   "source": [
    "# test\n",
    "sample_transformer = Transformer(2, 8500, 8000, max_length, 512, 8, 2048, rate=0.1)\n",
    "\n",
    "temp_input = tf.random.uniform((64, 26))\n",
    "temp_target = tf.random.uniform((64, 31))\n",
    "\n",
    "predictions, attention_weights = sample_transformer(\n",
    "    temp_input, temp_target, training = False, encoder_padding_mask = None,\n",
    "    decoder_mask = None, encoder_decoder_padding_mask = None)\n",
    "\n",
    "print(predictions.shape)\n",
    "for key in attention_weights:\n",
    "    print(key, attention_weights[key].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "aWqQTNxaW4Q5"
   },
   "source": [
    "## Train\n",
    "### initializes model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3mqkrf63W4Q6"
   },
   "outputs": [],
   "source": [
    "# 定义超参数\n",
    "num_layers = 4\n",
    "d_model = 128\n",
    "dff = 512\n",
    "num_heads = 8\n",
    "dropout_rate = 0.1\n",
    "\n",
    "input_vocab_size = pt_tokenizer.vocab_size + 2\n",
    "target_vocab_size = en_tokenizer.vocab_size + 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hpE5TvGiW4Q-"
   },
   "outputs": [],
   "source": [
    "# model\n",
    "transformer = Transformer(num_layers, input_vocab_size, target_vocab_size,\n",
    "                          max_length, d_model, num_heads, dff, dropout_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lCjA-1CiW4RA"
   },
   "source": [
    "### 自定义学习率\n",
    "learning_rate 先增后减\n",
    "\n",
    "$$\n",
    "lrate = (d\\_model^{-0.5}) * min(step\\_num^{-0.5}, step\\_num * warm\\_up\\_steps^{-1.5})\n",
    "$$ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GpjOodrIW4RB"
   },
   "outputs": [],
   "source": [
    "class CustomizedSchedule(keras.optimizers.schedules.LearningRateSchedule):\n",
    "    def __init__(self, d_model, warmup_steps = 4000):\n",
    "        super(CustomizedSchedule, self).__init__()\n",
    "        self.d_model = d_model\n",
    "        self.warmup_steps = warmup_steps\n",
    "        \n",
    "    def __call__(self, step):\n",
    "        arg1 = tf.math.rsqrt(step)\n",
    "        arg2 = step * (self.warmup_steps ** (-1.5))\n",
    "        arg3 = tf.math.rsqrt(tf.cast(self.d_model, dtype=tf.float32))\n",
    "        return arg3 * tf.math.minimum(arg1, arg2)\n",
    "    \n",
    "learning_rate = CustomizedSchedule(d_model)\n",
    "optimizer = keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 297
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 256203,
     "status": "ok",
     "timestamp": 1581685063312,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "198kXakZW4RD",
    "outputId": "0eb0a378-5fb4-46cb-e248-9afe8e53d2ea"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'Train step')"
      ]
     },
     "execution_count": 47,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEGCAYAAACtqQjWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU9bn48c+TnYQkZGMNkBDCElxQ\nI651wwVtr9z2agv13kvVSm/Vrrd1+fX3s73+6r3a9la7aPtz11YFSrWlve5LXaoiAQUBRTIDQtgy\nCRBJgIQkz++P8w0McZJMkpnMJPO8X6+8cuZ7zvmeZyaQJ+d8v+c5oqoYY4wxkZAU6wCMMcYMHZZU\njDHGRIwlFWOMMRFjScUYY0zEWFIxxhgTMSmxDiCWCgsLtaSkJNZhGGPMoLJy5co6VS0KtS6hk0pJ\nSQlVVVWxDsMYYwYVEfm4q3V2+csYY0zEWFIxxhgTMZZUjDHGRIwlFWOMMRFjScUYY0zERDWpiMgc\nEdkgItUiclOI9ekistitXy4iJUHrbnbtG0TkoqD2B0WkVkTWdnHMfxcRFZHCaLwnY4wxXYtaUhGR\nZOBu4GKgApgvIhWdNrsa2KOqk4E7gTvcvhXAPGAGMAe4x/UH8LBrC3XM8cCFwJaIvhljjDFhieaZ\nyiygWlX9qtoCLALmdtpmLvCIW14KzBYRce2LVLVZVTcB1a4/VPU1YHcXx7wTuAEYkvX8VZUlK7bS\n2Nwa61CMMSakaCaVccDWoNc1ri3kNqraCjQABWHuexQRmQtsU9XVPWy3UESqRKQqEAiE8z7ixntb\n93LDH9dw49I1sQ7FGGNCGhID9SKSCfwv4JaetlXVe1W1UlUri4pCVhmIW1t27wfghQ92xTgSY4wJ\nLZpJZRswPuh1sWsLuY2IpAC5QH2Y+wYrA0qB1SKy2W2/SkRG9yP+uOMLNAHQ0trOVpdgjDEmnkQz\nqawAykWkVETS8Abel3XaZhmwwC1fBrys3vONlwHz3OywUqAceKerA6nq+6o6UlVLVLUE73LZiaq6\nM7JvKbZ8gUZEvOVn1u6IbTDGGBNC1JKKGyO5HngO+ABYoqrrRORWEbnUbfYAUCAi1cB3gZvcvuuA\nJcB64FngOlVtAxCRJ4C3gKkiUiMiV0frPcQbf6CJs6cUMWNsDs+sHVL50hgzRES1SrGqPg083ant\nlqDlg8DlXex7G3BbiPb5YRy3pLexxrv2dmVTXSOnlxVwckk+P31uAzsaDjAmd1isQzPGmMOGxEB9\nItjecICDh9qZVJTFnGO8oaJn7WzFGBNnLKkMEn43SF9WNJyyouFMG53NX1Zvj3FUxhhzNEsqg4Qv\n0AjApKIsAObOHMeqLXv5uL4plmEZY8xRLKkMEv5AE9kZKRQNTwdg7syxiMCf3rWzFWNM/LCkMkj4\nAo1MKhqOuDnFY0cM49TSAp56twZvFrYxxsSeJZVBwh9ooqww66i2z584js31+3l3694YRWWMMUez\npDIINDa3svOTg5SNHH5U+8XHjCY9JYk/vdtdsQFjjBk4llQGgU1u5tekTmcq2RmpXFAxir+s3k5z\na1ssQjPGmKNYUhkE/HXezK/OZyoAl1eOZ8/+Qzy/zopMGmNiz5LKIOCrbSRJYGJB5qfWfWZyIcV5\nw3h8uT2XzBgTe5ZUBgFfXRPFeZmkpyR/al1SkjB/1gTe8tfjd/eyGGNMrFhSGQR8tY2UFWV1uf7y\nymJSkoRFK7Z2uY0xxgwESypxrr1d2VzfxKSiT4+ndBiZncH500exdGWNDdgbY2LKkkqc6ygkWdZN\nUgH48ikT2N3UYkUmjTExZUklznU87XFSN5e/AM6cXEhpYRYP/n2z3WFvjIkZSypxrmPwvaczlaQk\n4cozSli9dS+rtuwZiNCMMeZTLKnEOV+gkeyMFAqHp/W47WUnFZM7LJX7X980AJEZY8ynWVKJc/5A\n01GFJLuTmZbC/FkTeG7dTrbu3j8A0RljzNEsqcQ5f6Cp2+nEnS04fSJJIjz85uboBWWMMV2IalIR\nkTkiskFEqkXkphDr00VksVu/XERKgtbd7No3iMhFQe0PikitiKzt1NdPReRDEVkjIk+JyIhovreB\ncLiQZA/jKcHG5A7jkmPHsHjFVhr2H4pidMYY82lRSyoikgzcDVwMVADzRaSi02ZXA3tUdTJwJ3CH\n27cCmAfMAOYA97j+AB52bZ29AByjqscBHwE3R/QNxcCmw48QDv9MBeDr55TR2NzKQ2/a2IoxZmBF\n80xlFlCtqn5VbQEWAXM7bTMXeMQtLwVmizd4MBdYpKrNqroJqHb9oaqvAbs7H0xVn1fVVvfybaA4\n0m9ooB15hHD4ZyoA08fkcP70UTz0983sO2hnK8aYgRPNpDIOCK4bUuPaQm7jEkIDUBDmvt25Cngm\n1AoRWSgiVSJSFQgEetHlwPMHui4k2ZNvzp5Mw4FD/O7tj6MQmTHGhDbkBupF5AdAK/BYqPWqeq+q\nVqpqZVFR0cAG10u+QBPj80MXkuzJccUjOHtKEfe/von9La0972CMMREQzaSyDRgf9LrYtYXcRkRS\ngFygPsx9P0VEvgJ8DrhCh8Bt5b5A46cezNUb3zhvMrubWnjsbSuLb4wZGNFMKiuAchEpFZE0vIH3\nZZ22WQYscMuXAS+7ZLAMmOdmh5UC5cA73R1MROYANwCXquqgv0mjvV3ZVNfUq5lfnVWW5HPm5EJ+\n86rPxlaMMQMiaknFjZFcDzwHfAAsUdV1InKriFzqNnsAKBCRauC7wE1u33XAEmA98Cxwnaq2AYjI\nE8BbwFQRqRGRq11fvwaygRdE5D0R+W203ttA2Lb3AM2t7b0epO/sxjnT2N3Uwn2v+SMUmTHGdC0l\nmp2r6tPA053abglaPghc3sW+twG3hWif38X2k/sVbJzx1/VtOnFnxxbn8tnjxnD/G5v4l9NKKMpO\nj0R4xhgT0pAbqB8qfLV9m04cyvcunEpLazu/enljv/syxpjuWFKJU/668AtJ9qS0MIsvnTyex5dv\nYbM7AzLGmGiwpBKnvJpf4RWSDMe3ZpeTnpLEj//ng4j0Z4wxoVhSiVO+QGOPD+bqjZE5GXxjdjkv\nfrCLv22ojVi/xhgTzJJKHGpsbmXXJ839mk4cypVnlFBamMWtf1lPS2t7RPs2xhiwpBKXjjztMXJn\nKgDpKcnc8g8V+OuaeNiKTRpjosCSShzyH34ufWTPVADOnTqS2dNG8osXN7Kz4WDE+zfGJDZLKnHI\n149CkuG45R8qaFPl//x5LUOgmo0xJo5YUolD/n4UkgzHxIIsvnP+FF5Yv4tn1u6MyjGMMYnJkkoc\n8gUaIz5I39nVZ5ZyzLgcbvnzOntCpDEmYiypxJmOQpL9qU4cjpTkJO74p+PYs7+F255eH9VjGWMS\nhyWVONNRSLJsZHTPVABmjM1l4VmTWFJVwyt274oxJgIsqcSZw48QjvKZSodvzS5n6qhsbli6hvrG\n5gE5pjFm6LKkEmeiOZ04lIzUZO6aN5OG/Ye4+cn3bTaYMaZfLKnEGX9dIzkRKiQZruljcrhhzlSe\nX7+LJVVbB+y4xpihx5JKnPHVNjEpgoUkw3XVGaWcXlbAf/xl/eE7+o0xprcsqcQZf130pxOHkpQk\n/PcXjyc9JYlrH1vFgZa2AY/BGDP4WVKJI/sOHmLXJ80RrU7cG2Nyh3Hnl2ayYdc+/vef7G57Y0zv\nWVKJI5si9Ajh/jhn6ki+cV45f1xVw+IVNr5ijOmdqCYVEZkjIhtEpFpEbgqxPl1EFrv1y0WkJGjd\nza59g4hcFNT+oIjUisjaTn3li8gLIrLRfc+L5nuLBt/h6sQDf/kr2Ldml/OZ8kJuWbaOtdsaYhqL\nMWZwiVpSEZFk4G7gYqACmC8iFZ02uxrYo6qTgTuBO9y+FcA8YAYwB7jH9QfwsGvr7CbgJVUtB15y\nrwcVf6CJJIEJUSokGa7kJOGuL82kMCuNax6tonafVTM2xoQnmmcqs4BqVfWraguwCJjbaZu5wCNu\neSkwW7xpT3OBRararKqbgGrXH6r6GrA7xPGC+3oE+MdIvpmB4A80MSGKhSR7o2B4OvctqGTv/kMs\nfHQlBw/ZwL0xpmfRTCrjgOCL8jWuLeQ2qtoKNAAFYe7b2ShV3eGWdwKjQm0kIgtFpEpEqgKBQDjv\nY8B4jxCO7aWvYDPG5nLXvJm8t3UvNyxdYwP3xpgeDcmBevV++4X8Daiq96pqpapWFhUVDXBkXWtz\nhSRjOUgfykUzRnPDnKksW72dX71cHetwjDFxLppJZRswPuh1sWsLuY2IpAC5QH2Y+3a2S0TGuL7G\nAIOqQuJ2V0gyns5UOnz97DK+cOI4fv7CRyyxGWHGmG5EM6msAMpFpFRE0vAG3pd12mYZsMAtXwa8\n7M4ylgHz3OywUqAceKeH4wX3tQD4cwTew4AZ6EKSvSEi3P6F4zhrShE3PbmGF9bvinVIxpg4FbWk\n4sZIrgeeAz4AlqjqOhG5VUQudZs9ABSISDXwXdyMLVVdBywB1gPPAtepahuAiDwBvAVMFZEaEbna\n9XU7cIGIbATOd68HjY5CkgNR8r4v0lKS+M0VJ3Js8Qiuf3wV72wKNVfCGJPoJJEHXysrK7WqqirW\nYQDwg6fe5y+rt7P6hxcOeN2v3tjd1MJlv32TwL5mFi88jYqxObEOyRgzwERkpapWhlo3JAfqByN/\noImykQNfSLK38rPSePSqWQxPT+GK+9/mgx2fxDokY0wcsaQSJ3yBRiYVxuelr86K8zJ54ppTSU9J\n5or7l7Nh575Yh2SMiROWVOLAvoOHqN0Xu0KSfVFSmMUTC08lNVn48n1vs3GXJRZjjCWVuHB4kD4O\npxN3p7Qwi8evOZXkJGH+fXYpzBhjSSUu+Os6CkkOnjOVDmVFw3li4amkJCXxpf/3Fis/tllhxiQy\nSypxwB9oIjlJYl5Isq/Kioaz9OunUTA8nSvuX87fNgyq+06NMRHUY1IRkUwR+T8icp97XS4in4t+\naInDF2hkfN6wuCgk2VfFeZks+dppTCoczjWPVvGX1dtjHZIxJgbCOVN5CGgGTnOvtwE/jlpECcgf\naBp04ymhFGWns+hrp3LC+Dy+uehd7n3NZ0UojUkw4SSVMlX9CXAIQFX3A/F9M8Ug0tau+OuaBtXM\nr+7kZKTy6NWzuOSYMfzn0x/yv55ay6G29liHZYwZIClhbNMiIsNwVX9FpAzvzMVEwPa9B2iJ00KS\nfZWRmsyv5p/AxIJM7vmbj5o9+7n7ihPJyUiNdWjGmCgL50zlR3j1t8aLyGN4T1W8MZpBJZJ4eYRw\npCUlCTfMmcZPLjuOt3z1/NM9b7K5rinWYRljoqzHpKKqzwNfAL4CPAFUquorUY4rYfjcPSpD5fJX\nZ1+sHM+jV88i0NjMP/z6DV76wCocGzOUhTP76yVVrVfV/1HVv6pqnYi8NBDBJQJ/oJHcYakUZKXF\nOpSoOb2skL9cfyYTCzK5+pEqfv78BtrabQDfmKGoy6QiIhkikg8UikieiOS7rxJ6frSvCZP3COGs\nuC8k2V/j8zNZ+m+nc/lJxfzy5WquengFe5paYh2WMSbCujtT+RqwEpjmvnd8/Rn4dfRDSwz+QNOg\nKSTZXxmpyfzksuO47fPH8Kavjkt++Tpv++tjHZYxJoK6TCqq+gtVLQW+p6qTVLXUfR2vqpZUIqCj\nkGTZyKE5nhKKiHDFKRN58utnkJGazPz73ubnz2+g1aYdGzMk9DilWFV/JSLHABVARlD7o9EMLBF0\nFJJMlDOVYMcW5/LXb5zJD5et45cvV/N3Xz13fWkm4/MHZ6kaY4wnnIH6HwK/cl/nAj8BLu12JxOW\njkKSkxPoTCVYVnoKP7v8eH45/wQ+2rmPS37xOkuqttpd+MYMYuHcp3IZMBvYqapXAscDuVGNKkH4\nal0hyfzETCodLj1+LE9/6zNMH5vDDUvX8JWHVrB974FYh2WM6YNwksoBVW0HWkUkB6gFxkc3rMTg\nr2tkQn4maSlWLHp8fiaLrjmV/7h0Bu9s2s1Fd77G4hVb7KzFmEEmnN9mVSIyArgPb/bXKuCtcDoX\nkTkiskFEqkXkphDr00VksVu/3E1X7lh3s2vfICIX9dSniMwWkVUi8p6IvCEik8OJMZZ8tU1MKkzs\ns5RgSUnCgtNLeO7bZ1ExNocb//g+//rgO3YnvjGDSLdJRbybJ/5LVfeq6m+BC4AF7jJYt0QkGbgb\nuBhvkH++iFR02uxqYI+qTgbuBO5w+1YA84AZwBzgHhFJ7qHP3wBXqOpM4HHgf/f47mOorV3ZVD90\nCklG0oSCTJ645lRunTuDd7fs5cK7XuMXL26kubUt1qEZY3rQbVJR79rD00GvN6vqmjD7ngVUq6pf\nVVuARcDcTtvMBR5xy0uB2S6RzQUWqWqzqm4Cql1/3fWpQI5bzgXi+oEeHYUkh1rNr0hJShL+9bQS\nXvr3s7mwYhR3vvgRc+56nTc21sU6NGNMN8K5/LVKRE7uQ9/jgK1Br2v49J34h7dR1VagASjoZt/u\n+vwq8LSI1AD/AtweKigRWSgiVSJSFQgE+vC2IqPaFZIcStWJo2FUTga//vKJPHrVLFSVf35gOdc/\nvoptNpBvTFwKJ6mcArwlIj4RWSMi74tIuGcrA+k7wCWqWoz3YLGfh9pIVe9V1UpVrSwqKhrQAIN1\n3KMyGJ9LHwtnTSni2W+fxbfPL+eF9bs472d/46fPfUhjc2usQzPGBAnneSoX9bxJSNs4epZYsWsL\ntU2NiKTgXbaq72HfT7WLSBFwvKoud+2L8cr1xy2fKySZP4QLSUZaRmoy3z5/CpdXjuenz37I3a/4\nWFJVw/cunMJlJ40nOWlo108zZjAIp/T9x6G+wuh7BVAuIqUikoY38L6s0zbLgAVu+TLgZTeOswyY\n52aHlQLlwDvd9LkHyBWRKa6vC4APwogxZvwJUkgyGsaNGMZd807gqWtPZ0J+Jjf+8X0++8vX+duG\nWpuCbEyMhXOm0ieq2ioi1wPPAcnAg6q6TkRuBapUdRnwAPA7EakGduMlCdx2S4D1QCtwnaq2AYTq\n07VfA/xRRNrxksxV0XpvkeALNHH2lNhdfhsKTpiQx9J/O42n39/J7c9+wFceWsHJJXl878KpnDKp\nINbhGZOQJJH/squsrNSqqqoBP+6+g4c49kfPc8OcqVx7TtzfTjMotLS2s7hqK79+eSO7PmnmM+WF\nfO/CqRw/fkSsQzNmyBGRlapaGWqd3codA0cG6W3mV6SkpSTxL6dO5NXvn8sPLpnOuu2fMPfuv3PN\no1Wsqdkb6/CMSRg9Xv4SkX1494AEawCqgH9XVX80AhvKjjyX3mZ+RVpGajLXnDWJ+adM4ME3NnHf\n635eWL+Lz5QXct25kzmlNN/GsYyJonDGVO7Cux/kcUDwxj3K8Mq1PAicE63ghip/wApJRtvw9BS+\nObucK88o4fdvb+GBN/zMu/dtTpqYx3XnlnHu1JGWXIyJgnAuf12qqv9PVfep6ieqei9wkaouBvKi\nHN+Q5AtYIcmBkp2RytfPKeONG8/j1rkz2NlwkKseruLiX7zOU+/W0NJqDwczJpLC+a22X0S+KCJJ\n7uuLwEG3LnFH+fvBe4SwnaUMpIzUZP71tBL+9v1z+Nnlx3OorZ3vLF7NGXe8zK9e2kh9Y3OsQzRm\nSAgnqVyBV/akFtjllv9ZRIYB10cxtiGpo5Bk2UgbpI+F1OQkLjupmBe+czYPX3ky08fk8N8vfMRp\nt7/MjUvX8OHOT2IdojGDWjiPE/YD/9DF6jciG87Qt22PV0jSzlRiKylJOGfqSM6ZOpKNu/bx0Jub\neXJVDYurtnJ6WQH/fOpELqgYRWqyXaI0pjfCmf1VBFwDlARvr6pxfXNhvPK5RwjbmUr8KB+VzX9+\n/li+f+FUnlixhd+/9THXPraKwuHpfLGymPmzJjA+PzPWYRozKIQz++vPwOvAi4A90KKffLWuOrGd\nqcSdvKw0rj1nMl87q4xXP6rl8eVb+O2rPn7zqo/PlBfx5VkTmD19pJ29GNONcJJKpqreGPVIEoS/\nrokRmVZIMp4lJwnnTRvFedNGsX3vARav2MriFVv5t9+vpCg7nX+cOZYvnFjM9DE5PXdmTIIJJ6n8\nVUQuUdWne97U9MRX28ikQiskOViMHTGM71wwhW+cN5lXNgT4Q9VWHn5zM/e9vomKMTl84cRxzJ05\njqLs9FiHakxc6LH2l7ujPgtoBg7h3QCpqjro/0yLRe2vk297kbOnFPGzy48f0OOayNnd1MJfVm/n\nyVU1rK5pIDlJOHtKEV84cRznTx9FRmpyrEM0Jqq6q/0Vzuyv7MiHlJg+OXiIwL5mq/k1yOVnpbHg\n9BIWnF7Cxl37ePLdbTy1ahsvf1hLVloy51eM4rPHjuHsqUWkp1iCMYmly6QiItNU9UMROTHUelVd\nFb2whqaOQpKTrObXkFE+Kpsb50zjexdO5W1/PX9ds51n1u7kz+9tJzs9hQtmjOJzx43hzMlFVkHB\nJITuzlS+CywE/jvEOgXOi0pEQ5j/cCFJO1MZapKThDMmF3LG5EJunXsMb/rq+evq7Ty3bidPrtpG\nTkYKF80YzcXHjub0skK7RGaGrC6TiqoudN/PHbhwhjZfoNEVkrR7Hoay1OQkzp5SxNlTirjt88fy\nRnWAv67ewTNrd/KHlTVkpiVz9pQiLqgYxXnTRjIi02YCmqEjrCc/isjpfPrmx0ejFNOQ5Q80WSHJ\nBJOWknR4enJzaxtv+ep5fv0uXly/i2fW7iQ5SZhVks8FFaO4oGKU3WRpBr1wZn/9Dq/U/XscuflR\nVfWbUY4t6gZ69teFd77KhPxM7l9w8oAd08Sn9nZlzbYGXli/k+fX7WKjuyl22uhsVz6miJMm5tmN\nliYu9Wv2F1AJVGgiP3c4Atralc31+zln6shYh2LiQFKSMHP8CGaOH8H3L5rG5romXli/ixc/2MX9\nr/v57as+hqencMbkAs6ZOpKzpxQxdsSwWIdtTI/CSSprgdHAjt52LiJzgF8AycD9qnp7p/XpwKPA\nSUA98CVV3ezW3QxcjXd29E1Vfa67PsW7m/DHwOVun9+o6i97G3O0dBSStKc9mlBKCrO45qxJXHPW\nJPYdPMTfq+t59aMAr26o5bl1uwCYMmo450wdyVnlRVSW5Nlgv4lL4SSVQmC9iLyDdwMkAKp6aXc7\niUgycDdwAd6TI1eIyDJVXR+02dXAHlWdLCLzgDuAL4lIBd4TJmcAY4EXRWSK26erPr8CjAemqWq7\niMTVKUHHI4Qn2cwv04PsjFTmHDOaOceMRlXZWNvI3zbU8upHAR76+ybufc1PWkoSlRPzOL2sgNMn\nF3LcuFxS7FKZiQPhJJUf9bHvWUB1xzPsRWQRMBcITipzg/pfCvzanXHMBRapajOwSUSqXX900+fX\ngS+rajuAqtb2Me6o8Nl0YtMHIsKUUdlMGZXNwrPKaGpu5W1/PW/6vK+fPf8RPP8Rw9NTOKU0n9PK\nCjhjciFTR2WTlGSlgMzA6zapuLONH/VxWvE4YGvQ6xrglK62UdVWEWkAClz72532HeeWu+qzDO8s\n5/NAAO+S2cYQ72kh3v03TJgwoffvqo98ASskafovKz2F2dNHMXv6KMArGfOWr543fXW86avnpQ+9\nv6UKstI4ZVI+J5d4X9PH5JBsScYMgG6Tiqq2iUi7iOSqasNABdVH6cBBVa0UkS8ADwKf6byRqt4L\n3Ave7K+BCs4faLRy9ybi8rPS+OxxY/jscWMA2L73AG/56vm7r47l/t08/f5OAIanp3DixDxmleRx\nckk+x48fYWMyJirCufzVCLwvIi8ATR2NYUwp3oY3xtGh2LWF2qZGRFKAXLwB++727aq9BnjSLT8F\nPNRDfAPKX9fEOVOKYh2GGeLGjhjGP51UzD+dVAx4SWbF5t3e16Y93uUyIC05ieOKc6ksyWdWaR4z\nx+fZWbSJiHCSypMc+WXdGyuAchEpxfvFPw/4cqdtlgELgLeAy4CXVVVFZBnwuIj8HG+gvhx4B69C\ncld9/gk4F9gEnA181IeYo6KjkKQN0puBNnbEMObO9MrzA+zd30LV5j2s2LybdzbvdtOXvRP2koJM\nZo4fwQkT8pg5fgTTx+TYjbqm18KpUvxIXzp2YyTXA8/hTf99UFXXicitQJWqLgMeAH7nBuJ34yUJ\n3HZL8AbgW4HrVLUNIFSf7pC3A4+JyHfwzq6+2pe4o6GjkKRNJzaxNiIzjfMrRnF+hTcmc6CljdU1\ne3lv617e27KXN331/Om97YBXDeDYcbku0Xj31IwbMcyeBWS6Fc4d9eXAfwEVQEZHu6pOim5o0TdQ\nd9T/cWUN//6H1bz43bOZbM+mN3FMVdnRcJD3tu7l3S17eHfLXt7f1kBzazsARdnpHDcul2Pc17Hj\nchmVk26JJsH09476h4AfAnfiXV66ErBz4l7w11khSTM4iAhjRwxj7IhhXHKsN/h/qK2dD3fs492t\ne3jPJZlXNtTS7v4eLRyezjHjcjg2KNGMyc2wRJOgwkkqw1T1JRERVf0Y+JGIrARuiXJsQ4avtomJ\nVkjSDFKpyUkcW5zLscW5/OtpXtv+llbWb/+EtdsaeH+b9/21jwKHE01BVhozxuVy7Lgcpo/JYdro\nHEoKMu0GzQQQTlJpFpEkYKMbz9gG2DWcXvDXNdqDucyQkpmWQmVJPpUl+YfbDrS08cFOl2hqGnh/\nWwO/ra6jzWWa9JQkpozKZtrobC/RjMlm+ugc8mzW2ZASTlL5FpAJfBP4v3iXwBZEM6ihpK1d2Vy3\nn3OtkKQZ4oalJXPihDxOnJB3uO3goTaqaxv5cOc+PtzxCR/u3MfLH9byh5U1h7cZlZPOtNFHksy0\nMdmUFQ23Cs2DVDizv1YAiEi7ql4Z/ZCGlpo9+2lpa7czFZOQMlKTDw/qBwvsa+bDnZ/w4Y59fOC+\nv+Wrp6XNmxCQmiyUFmZRPjKbspHDKR85nPJRwyktzCI9xW7ajGc9JhUROQ1v6u9wYIKIHA98TVWv\njXZwQ8GR6cR2xdCYDkXZ6RRlF/GZ8iM3BB9qa2dTXRMfuDOajbsaWb/jE55Zu+PwWE2SwMSCLCYH\nJZrJRdmUjcwiMy2sZw6aKPV0tq4AABOmSURBVAvnp3AXcBHejYqo6moROSuqUQ0hVp3YmPCkJicd\nLp45N6j94KE2NtU1sbG2kepd+9hY28jG2kZe+bCW1vYjt0QU5w2jfORwyoqGU1qURWlhFpMKh9uU\n5wEWVmpX1a2dfihtXW1rjmaFJI3pn4zUZKaP8WaRBTvU1s7H9U1s3NV4ONFs3LWPN331h++rAchM\nS6akIIvSoiwmFXrJpiPh5GamDvTbGfLCSSpb3TPqVURS8QbuP4huWEOHP9Bol76MiYLU5CQmj8xm\n8shsLg5qb29XdnxykE2BJjbVNeKva2JTXRNrtzXwzPtHLqWBV5CzNCjRlBZmMSE/kwkFmeRkWMLp\ni3CSyr/hPWlxHN504ucBG08Jky/QxLlTrZCkMQMlKUkYN2IY40YM48zywqPWtbS2s2X3fjbVeQln\nk0s4r28MsDRoRhrAiMxUJuZnMj4/k4kFmUw4vJzF6JwMe5RAF8KZ/VUHXBHcJiLfxhtrMd1oOHCI\nusZmyqw0izFxIS0lickjh7tySaOOWtfY3MqW+v1s2d3Elt37+bh+P1t27+f9bQ08u3bnUeM3aclJ\nFOcNOyrhdJzhjBsxjOwEPsvp63SJ72JJpUf+jkF6e46KMXFveHoKFWNzqBib86l1rW3t7Gg4eFSy\n6Ug+q7bsYd/B1qO2z8lIoTgvk3F53hlTcZ73NW6E15aXmTpkJw/0NakMzU8jwjqmE9vML2MGt5Tk\nJMa7y19nTD56narScODQ4WSzbe8Btu05wLa9B9hSv583q+toajl6blNmWrJ3ia5TsinOG0bxiGEU\nDk8ftI+D7mtSGbAnJg5mvkAjKUnCxAIrJGnMUCUijMhMY0RmGsePH/Gp9R1Jp2bPAWpcsvGSzn5q\n9hzgva172bv/0FH7pCUnMTo3g9G5GYzNzWB07jDGHH49jNG5GRRkpcVl4ukyqYjIPkInDwGGRS2i\nIcQfaGJCfqaVmzAmgQUnnc6VBTo0Nreyfe8BavbsZ9ueA9TsPcDOhoPs2HuQlVv2sLNhB4fajv51\nnJosjMo5kmQ6ks4Yl4DG5GbE5Iyny6SiqtkDGchQ5BWStEtfxpjuDU9POXzjZyjt7Up9U4uXaBoO\nsPOTg2zfe5CdDQcOP//m2bUHD5e56ZCS5CWe0bkZjMpJ95ZzMhiVk8HpZQWMzMkIebz+sLoGUWKF\nJI0xkZKUJK60TTrHFoc+21FVdje1sKPhIDsajiQcb/kgH+7cx6sbAofHdx69apYllcGko5Ck3fho\njBkIIkLB8HQKhqd3eZkNYN/BQ+z6pJkxuZFPKGBJJWqO1Pyy6cTGmPiRnZEa1ftoojqCLCJzRGSD\niFSLyE0h1qeLyGK3frmIlAStu9m1bxCRi3rR5y9FpDFa7ylcNp3YGJOIopZURCQZuBu4GKgA5otI\nRafNrgb2qOpk4E7gDrdvBTAPmAHMAe4RkeSe+hSRSiCPOOALNJFnhSSNMQkmmmcqs4BqVfWraguw\nCI6qaI17/YhbXgrMFu8207nAIlVtVtVNQLXrr8s+XcL5KXBDFN9T2HwBm/lljEk80Uwq44CtQa9r\nXFvIbVS1FWgACrrZt7s+rweWqeqO7oISkYUiUiUiVYFAoFdvqDf8gSbKbDzFGJNghsRdeSIyFrgc\n+FVP26rqvapaqaqVRUXRqR7cUUjSzlSMMYkmmkllGzA+6HWxawu5jYikALlAfTf7dtV+AjAZqBaR\nzUCmiFRH6o30lhWSNMYkqmgmlRVAuYiUikga3sD7sk7bLAMWuOXLgJdVVV37PDc7rBQoB97pqk9V\n/R9VHa2qJapaAux3g/8x4et4Lr2VvDfGJJio3aeiqq0icj3wHJAMPKiq60TkVqBKVZcBDwC/c2cV\nu/GSBG67JcB6oBW4TlXbAEL1Ga330Fd+V0hyQr4VkjTGJJao3vyoqk8DT3dquyVo+SDeWEiofW8D\nbgunzxDbxPQUwR9oYkKBFZI0xiQe+60XBb5AI5MK7dKXMSbxWFKJsNa2dj6u30/ZSBukN8YkHksq\nEVaz54BXSNLOVIwxCciSSoT566yQpDEmcVlSibCOQpJW8t4Yk4gsqUSYL9BIXmYqeVZI0hiTgCyp\nRJgv0GRnKcaYhGVJJcL8gUYbTzHGJCxLKhHUsP8QdY0tVkjSGJOwLKlEkM/N/LLLX8aYRGVJJYKO\nPELYLn8ZYxKTJZUIskKSxphEZ0klgnyBRiskaYxJaPbbL4L8Np3YGJPgLKlESGtbO5vrm2w8xRiT\n0CypREjNngMcalMrJGmMSWiWVCKko5Cklbw3xiQySyoR4qt104ntTMUYk8AsqUSIv66R/Kw0KyRp\njEloUU0qIjJHRDaISLWI3BRifbqILHbrl4tISdC6m137BhG5qKc+ReQx175WRB4UkdRovrfOfLVN\nTCq0S1/GmMQWtaQiIsnA3cDFQAUwX0QqOm12NbBHVScDdwJ3uH0rgHnADGAOcI+IJPfQ52PANOBY\nYBjw1Wi9t1D8dVZI0hhjonmmMguoVlW/qrYAi4C5nbaZCzzilpcCs0VEXPsiVW1W1U1Ateuvyz5V\n9Wl1gHeA4ii+t6N0FJK0e1SMMYkumkllHLA16HWNawu5jaq2Ag1AQTf79tinu+z1L8Cz/X4HYfId\nfoSwJRVjTGIbigP19wCvqerroVaKyEIRqRKRqkAgEJEDHnmEsF3+MsYktmgmlW3A+KDXxa4t5DYi\nkgLkAvXd7NttnyLyQ6AI+G5XQanqvapaqaqVRUVFvXxLoflcIcnxVkjSGJPgoplUVgDlIlIqIml4\nA+/LOm2zDFjgli8DXnZjIsuAeW52WClQjjdO0mWfIvJV4CJgvqq2R/F9fYo/0MhEKyRpjDGkRKtj\nVW0VkeuB54Bk4EFVXScitwJVqroMeAD4nYhUA7vxkgRuuyXAeqAVuE5V2wBC9ekO+VvgY+Atb6yf\nJ1X11mi9v2C+QJONpxhjDFFMKuDNyAKe7tR2S9DyQeDyLva9DbgtnD5de1TfS1da29r5uL6J2dNH\nxuLwxhgTV+x6TT8dLiRpZyrGGGNJpb98gY7n0tvML2OMsaTST4efS2+FJI0xxpJKf/kCVkjSGGM6\nWFLpJ3/ACkkaY0wHSyr95As02iC9McY4llT6oWH/IeqbWqw6sTHGOJZU+qGjkKSdqRhjjMeSSj/4\najuqE9uZijHGgCWVfvHXNZGabIUkjTGmgyWVfvDVNjIh3wpJGmNMB/tt2A/+OiskaYwxwSyp9FFH\nIUkbpDfGmCMsqfTRVldI0gbpjTHmCEsqfeQP2HRiY4zpzJJKH1l1YmOM+TRLKn3kDzRRkJXGiEwr\nJGmMMR0sqfSRL9Bo4ynGGNOJJZU+8qoT23iKMcYEs6TSB3v3t1Df1ELZSDtTMcaYYFFNKiIyR0Q2\niEi1iNwUYn26iCx265eLSEnQuptd+wYRuainPkWk1PVR7fqM2mCHz572aIwxIUUtqYhIMnA3cDFQ\nAcwXkYpOm10N7FHVycCdwB1u3wpgHjADmAPcIyLJPfR5B3Cn62uP6zsqDk8nHmlJxRhjgkXzTGUW\nUK2qflVtARYBczttMxd4xC0vBWaLiLj2RararKqbgGrXX8g+3T7nuT5wff5jtN6YL+AKSeYNi9Yh\njDFmUIpmUhkHbA16XePaQm6jqq1AA1DQzb5dtRcAe10fXR0LABFZKCJVIlIVCAT68LagpCCTz58w\njhQrJGmMMUdJuN+KqnqvqlaqamVRUVGf+pg3awI/uez4CEdmjDGDXzSTyjZgfNDrYtcWchsRSQFy\ngfpu9u2qvR4Y4fro6ljGGGOiLJpJZQVQ7mZlpeENvC/rtM0yYIFbvgx4WVXVtc9zs8NKgXLgna76\ndPu84vrA9fnnKL43Y4wxIaT0vEnfqGqriFwPPAckAw+q6joRuRWoUtVlwAPA70SkGtiNlyRw2y0B\n1gOtwHWq2gYQqk93yBuBRSLyY+Bd17cxxpgBJN4f+YmpsrJSq6qqYh2GMcYMKiKyUlUrQ61LuIF6\nY4wx0WNJxRhjTMRYUjHGGBMxllSMMcZETEIP1ItIAPi4j7sXAnURDCdSLK7esbh6x+LqnXiNC/oX\n20RVDXn3eEInlf4QkaquZj/EksXVOxZX71hcvROvcUH0YrPLX8YYYyLGkooxxpiIsaTSd/fGOoAu\nWFy9Y3H1jsXVO/EaF0QpNhtTMcYYEzF2pmKMMSZiLKkYY4yJGEsqfSAic0Rkg4hUi8hNA3C8zSLy\nvoi8JyJVri1fRF4QkY3ue55rFxH5pYttjYicGNTPArf9RhFZ0NXxeojlQRGpFZG1QW0Ri0VETnLv\ntdrtK/2I60ciss19bu+JyCVB6252x9ggIhcFtYf82brHLSx37Yvdoxd6imm8iLwiIutFZJ2IfCse\nPq9u4orp5+X2yxCRd0RktYvtP7rrT7zHYyx27ctFpKSvMfcxrodFZFPQZzbTtQ/kv/1kEXlXRP4a\nD58VqmpfvfjCK7nvAyYBacBqoCLKx9wMFHZq+wlwk1u+CbjDLV8CPAMIcCqw3LXnA373Pc8t5/Uh\nlrOAE4G10YgF77k5p7p9ngEu7kdcPwK+F2LbCvdzSwdK3c8zubufLbAEmOeWfwt8PYyYxgAnuuVs\n4CN37Jh+Xt3EFdPPy20rwHC3nAosd+8vZH/AtcBv3fI8YHFfY+5jXA8Dl4XYfiD/7X8XeBz4a3ef\n/UB9Vnam0nuzgGpV9atqC7AImBuDOOYCj7jlR4B/DGp/VD1v4z0RcwxwEfCCqu5W1T3AC8Cc3h5U\nVV/De/ZNxGNx63JU9W31/rU/GtRXX+Lqylxgkao2q+omoBrv5xryZ+v+YjwPWBriPXYX0w5VXeWW\n9wEfAOOI8efVTVxdGZDPy8WjqtroXqa6L+2mv+DPcikw2x2/VzH3I66uDMjPUkSKgc8C97vX3X32\nA/JZWVLpvXHA1qDXNXT/HzISFHheRFaKyELXNkpVd7jlncCoHuKLZtyRimWcW45kjNe7yw8PirvM\n1Ie4CoC9qtra17jcpYYT8P7CjZvPq1NcEAefl7uc8x5Qi/dL19dNf4djcOsb3PEj/v+gc1yq2vGZ\n3eY+sztFJL1zXGEev68/y7uAG4B297q7z35APitLKoPDmap6InAxcJ2InBW80v1lExdzw+MpFuA3\nQBkwE9gB/HcsghCR4cAfgW+r6ifB62L5eYWIKy4+L1VtU9WZQDHeX8vTYhFHZ53jEpFjgJvx4jsZ\n75LWjQMVj4h8DqhV1ZUDdcxwWFLpvW3A+KDXxa4talR1m/teCzyF9x9tlztlxn2v7SG+aMYdqVi2\nueWIxKiqu9wvgnbgPrzPrS9x1eNdvkjp1N4jEUnF+8X9mKo+6Zpj/nmFiisePq9gqroXeAU4rZv+\nDsfg1ue640ft/0FQXHPcpURV1WbgIfr+mfXlZ3kGcKmIbMa7NHUe8Ati/Vn1NOhiX58aFEvBG1wr\n5cjg1YwoHi8LyA5afhNvLOSnHD3Y+xO3/FmOHiB8x7XnA5vwBgfz3HJ+H2Mq4egB8YjFwqcHKy/p\nR1xjgpa/g3fdGGAGRw9M+vEGJbv82QJ/4OjBz2vDiEfwro3f1ak9pp9XN3HF9PNy2xYBI9zyMOB1\n4HNd9Qdcx9GDz0v6GnMf4xoT9JneBdweo3/753BkoD62n1Vffqkk+hfezI6P8K71/iDKx5rkfpir\ngXUdx8O7FvoSsBF4MegfpgB3u9jeByqD+roKbxCuGriyj/E8gXdp5BDeNdarIxkLUAmsdfv8Glf1\noY9x/c4ddw2wjKN/af7AHWMDQbNsuvrZup/DOy7ePwDpYcR0Jt6lrTXAe+7rklh/Xt3EFdPPy+13\nHPCui2EtcEt3/QEZ7nW1Wz+przH3Ma6X3We2Fvg9R2aIDdi/fbfvORxJKjH9rKxMizHGmIixMRVj\njDERY0nFGGNMxFhSMcYYEzGWVIwxxkSMJRVjjDERY0nFmF4SkYKgqrQ75ejKvuFW431IRKb2M45J\nIjKvP30YE2k2pdiYfhCRHwGNqvqzTu2C9/+rPeSOkTn2+cD1qhpWsUZjBoKdqRgTISIyWbxnlDyG\nd6PqGBG5V0Sq3DM4bgna9g0RmSkiKSKyV0RuF+9ZHW+JyMgQfZ/n1r8nIqtEJAu4HTjXtX3T9fVz\n8Z77sUZEvur2PV+856c8456NcXe4z+owprcsqRgTWdOAO1W1Qr2abTepaiVwPHCBiFSE2CcXeFVV\njwfewrvjurPvAwvVK2h4FnAQr8TLK6o6U1V/CSzEKzA4C6/A4XUiMsHtfwrwdbxnZ0wnNo9rMAnA\nkooxkeVT1aqg1/NFZBWwCu+XeaikckBVn3HLK/FqmHX2d+AXIvINvOdutIXY5kLgSleefTkwAih3\n695W1c1uv0V4pVqMibiUnjcxxvRCU8eCiJQD3wJmqepeEfk9Xv2lzlqCltsI8f9SVX8sIsvwChW+\nLSKzQ/QjeMUDXzqq0Rt76Tx4aoOpJirsTMWY6MkB9gGfBD31r09EpExV16jqf+Gd9Ux1fWcHbfYc\ncG1H2XMRmSoiw9y6U0VkgogkA18E3uhrLMZ0x85UjImeVcB64EPgY7xLWH31PRH5DN4T/tYAz7v2\nZBFZDTyAVxV3AvCeG4ev5cjYyTt4ZdDL8CojL+tHLMZ0yaYUGzPE2dRjM5Ds8pcxxpiIsTMVY4wx\nEWNnKsYYYyLGkooxxpiIsaRijDEmYiypGGOMiRhLKsYYYyLm/wNIQcwJUvgs7QAAAABJRU5ErkJg\ngg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 画出learning_rate变化图示\n",
    "temp_learning_rate = CustomizedSchedule(d_model)\n",
    "\n",
    "plt.plot(temp_learning_rate(tf.range(40000, dtype=tf.float32)))\n",
    "plt.ylabel('Learning rate')\n",
    "plt.xlabel('Train step')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "LErE8M8sW4RG"
   },
   "source": [
    "### Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nSK0LAmdW4RH"
   },
   "outputs": [],
   "source": [
    "loss_object = keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction='none')\n",
    "\n",
    "def loss_function(real, pred):\n",
    "    # 去除padding，去噪声\n",
    "    mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
    "    loss_ = loss_object(real, pred)\n",
    "    \n",
    "    mask = tf.cast(mask, dtype=loss_.dtype)\n",
    "    loss_ *= mask\n",
    "    return tf.reduce_mean(loss_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "yY6LWLLGW4RJ"
   },
   "source": [
    "### 工具函数：mask创建\n",
    "问题：一个多头注意力，只能有一个mask，但是DecoderLayer上有两个mask，即look_ahead_mask,decoder_padding_mask，该怎么办？\n",
    "\n",
    "答：将两个mask做与操作，只要其中一个有mask，就mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "joYIxWU1W4RJ"
   },
   "outputs": [],
   "source": [
    "def create_masks(inp, tar):\n",
    "    \"\"\"\n",
    "    Encoder:\n",
    "      - encoder_padding_mask (self attention of EncoderLayer)\n",
    "    Decoder:\n",
    "      - look_ahead_mask (self attention of DecoderLayer)\n",
    "      - encoder_decoder_padding_mask (encoder-decoder attention of DecoderLayer)\n",
    "      - decoder_padding_mask (self attention of DecoderLayer)\n",
    "    \"\"\"\n",
    "    encoder_padding_mask = create_padding_mask(inp)\n",
    "    encoder_decoder_padding_mask = create_padding_mask(inp)\n",
    "    \n",
    "    look_ahead_mask = create_look_ahead_mask(tf.shape(tar)[1])\n",
    "    decoder_padding_mask = create_padding_mask(tar)\n",
    "    decoder_mask = tf.maximum(decoder_padding_mask, look_ahead_mask)\n",
    "    \n",
    "#     print(encoder_padding_mask.shape)\n",
    "#     print(encoder_decoder_padding_mask.shape)\n",
    "#     print(look_ahead_mask.shape)\n",
    "#     print(decoder_padding_mask.shape)\n",
    "#     print(decoder_mask.shape)\n",
    "    \n",
    "    return encoder_padding_mask, decoder_mask, encoder_decoder_padding_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 264196,
     "status": "ok",
     "timestamp": 1581685075581,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "I9gjmgACW4RN",
    "outputId": "5f3c422c-7ffd-4db8-b899-229cad36abc0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 38)\n",
      "(64, 37)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor: shape=(64, 1, 1, 38), dtype=float32, numpy=\n",
       " array([[[[0., 0., 0., ..., 1., 1., 1.]]],\n",
       " \n",
       " \n",
       "        [[[0., 0., 0., ..., 1., 1., 1.]]],\n",
       " \n",
       " \n",
       "        [[[0., 0., 0., ..., 1., 1., 1.]]],\n",
       " \n",
       " \n",
       "        ...,\n",
       " \n",
       " \n",
       "        [[[0., 0., 0., ..., 1., 1., 1.]]],\n",
       " \n",
       " \n",
       "        [[[0., 0., 0., ..., 1., 1., 1.]]],\n",
       " \n",
       " \n",
       "        [[[0., 0., 0., ..., 1., 1., 1.]]]], dtype=float32)>,\n",
       " <tf.Tensor: shape=(64, 1, 37, 37), dtype=float32, numpy=\n",
       " array([[[[0., 1., 1., ..., 1., 1., 1.],\n",
       "          [0., 0., 1., ..., 1., 1., 1.],\n",
       "          [0., 0., 0., ..., 1., 1., 1.],\n",
       "          ...,\n",
       "          [0., 0., 0., ..., 1., 1., 1.],\n",
       "          [0., 0., 0., ..., 1., 1., 1.],\n",
       "          [0., 0., 0., ..., 1., 1., 1.]]],\n",
       " \n",
       " \n",
       "        [[[0., 1., 1., ..., 1., 1., 1.],\n",
       "          [0., 0., 1., ..., 1., 1., 1.],\n",
       "          [0., 0., 0., ..., 1., 1., 1.],\n",
       "          ...,\n",
       "          [0., 0., 0., ..., 1., 1., 1.],\n",
       "          [0., 0., 0., ..., 1., 1., 1.],\n",
       "          [0., 0., 0., ..., 1., 1., 1.]]],\n",
       " \n",
       " \n",
       "        [[[0., 1., 1., ..., 1., 1., 1.],\n",
       "          [0., 0., 1., ..., 1., 1., 1.],\n",
       "          [0., 0., 0., ..., 1., 1., 1.],\n",
       "          ...,\n",
       "          [0., 0., 0., ..., 1., 1., 1.],\n",
       "          [0., 0., 0., ..., 1., 1., 1.],\n",
       "          [0., 0., 0., ..., 1., 1., 1.]]],\n",
       " \n",
       " \n",
       "        ...,\n",
       " \n",
       " \n",
       "        [[[0., 1., 1., ..., 1., 1., 1.],\n",
       "          [0., 0., 1., ..., 1., 1., 1.],\n",
       "          [0., 0., 0., ..., 1., 1., 1.],\n",
       "          ...,\n",
       "          [0., 0., 0., ..., 1., 1., 1.],\n",
       "          [0., 0., 0., ..., 1., 1., 1.],\n",
       "          [0., 0., 0., ..., 1., 1., 1.]]],\n",
       " \n",
       " \n",
       "        [[[0., 1., 1., ..., 1., 1., 1.],\n",
       "          [0., 0., 1., ..., 1., 1., 1.],\n",
       "          [0., 0., 0., ..., 1., 1., 1.],\n",
       "          ...,\n",
       "          [0., 0., 0., ..., 1., 1., 1.],\n",
       "          [0., 0., 0., ..., 1., 1., 1.],\n",
       "          [0., 0., 0., ..., 1., 1., 1.]]],\n",
       " \n",
       " \n",
       "        [[[0., 1., 1., ..., 1., 1., 1.],\n",
       "          [0., 0., 1., ..., 1., 1., 1.],\n",
       "          [0., 0., 0., ..., 1., 1., 1.],\n",
       "          ...,\n",
       "          [0., 0., 0., ..., 1., 1., 1.],\n",
       "          [0., 0., 0., ..., 1., 1., 1.],\n",
       "          [0., 0., 0., ..., 1., 1., 1.]]]], dtype=float32)>,\n",
       " <tf.Tensor: shape=(64, 1, 1, 38), dtype=float32, numpy=\n",
       " array([[[[0., 0., 0., ..., 1., 1., 1.]]],\n",
       " \n",
       " \n",
       "        [[[0., 0., 0., ..., 1., 1., 1.]]],\n",
       " \n",
       " \n",
       "        [[[0., 0., 0., ..., 1., 1., 1.]]],\n",
       " \n",
       " \n",
       "        ...,\n",
       " \n",
       " \n",
       "        [[[0., 0., 0., ..., 1., 1., 1.]]],\n",
       " \n",
       " \n",
       "        [[[0., 0., 0., ..., 1., 1., 1.]]],\n",
       " \n",
       " \n",
       "        [[[0., 0., 0., ..., 1., 1., 1.]]]], dtype=float32)>)"
      ]
     },
     "execution_count": 50,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test \n",
    "temp_inp, temp_tar = iter(train_dataset.take(1)).next()\n",
    "\n",
    "print(temp_inp.shape)\n",
    "print(temp_tar.shape)\n",
    "create_masks(temp_inp, temp_tar)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "39u0wA5WW4RQ"
   },
   "source": [
    "### 模型保存"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "E7-t_QBQW4RQ"
   },
   "outputs": [],
   "source": [
    "checkpoint_path = os.path.join('/content/drive/My Drive/Colab Notebooks/transformer')\n",
    "if not os.path.exists(checkpoint_path):\n",
    "    os.mkdir(checkpoint_path)\n",
    "    print(checkpoint_path)\n",
    "ckpt = tf.train.Checkpoint(transformer=transformer, optimizer=optimizer)\n",
    "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=1)\n",
    "\n",
    "# 如果检查点存在，则恢复最新的检查点\n",
    "if ckpt_manager.latest_checkpoint:\n",
    "    ckpt.restore(ckpt_manager.latest_checkpoint)\n",
    "    print('Latest checkpoint restored!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YfnsD-ZWW4RS"
   },
   "source": [
    "### 训练Train step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OnK6MF8sW4RT"
   },
   "outputs": [],
   "source": [
    "# 方便可视训练过程，不是真正的NMT的训练准确度\n",
    "train_loss = keras.metrics.Mean(name = 'train_loss')\n",
    "train_accuracy = keras.metrics.SparseCategoricalAccuracy(name = 'train_accuracy')\n",
    "\n",
    "@tf.function(experimental_relax_shapes=True)\n",
    "def train_step(inp, tar):\n",
    "    # 把目标数据切分成decoder input和decoder output\n",
    "    tar_inp = tar[:, :-1]\n",
    "    tar_real = tar[:, 1:]\n",
    "    # 获取mask\n",
    "    encoder_padding_mask, decoder_mask, encoder_decoder_padding_mask = create_masks(inp, tar_inp)\n",
    "    \n",
    "    # 计算梯度\n",
    "    with tf.GradientTape() as tape:\n",
    "        predictions, _ = transformer(inp, tar_inp, True, encoder_padding_mask,\n",
    "                                     decoder_mask, encoder_decoder_padding_mask)\n",
    "        loss = loss_function(tar_real, predictions)\n",
    "        \n",
    "    gradients = tape.gradient(loss, transformer.trainable_variables)\n",
    "    optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))\n",
    "    \n",
    "    # 累积loss 和 accuracy\n",
    "    train_loss(loss)\n",
    "    train_accuracy(tar_real, predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1753496,
     "status": "ok",
     "timestamp": 1581686831208,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "jTUtZifZW4RW",
    "outputId": "69cf34f0-b462-4f79-8a4f-3d6ac9619933"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 Batch 0 Loss 3.9077 Accuracy 0.0000\n",
      "Epoch 1 Batch 100 Loss 4.1565 Accuracy 0.0088\n",
      "Epoch 1 Batch 200 Loss 4.0654 Accuracy 0.0167\n",
      "Epoch 1 Batch 300 Loss 3.9126 Accuracy 0.0227\n",
      "Epoch 1 Batch 400 Loss 3.7483 Accuracy 0.0299\n",
      "Epoch 1 Batch 500 Loss 3.6023 Accuracy 0.0363\n",
      "Epoch 1 Batch 600 Loss 3.4824 Accuracy 0.0434\n",
      "Epoch 1 Batch 700 Loss 3.3732 Accuracy 0.0503\n",
      "Epoch 1 Loss 3.3711 Accuracy 0.0504\n",
      "Time taken for 1 epoch: 110.85187244415283 secs\n",
      "\n",
      "Epoch 2 Batch 0 Loss 2.5038 Accuracy 0.0909\n",
      "Epoch 2 Batch 100 Loss 2.5583 Accuracy 0.1050\n",
      "Epoch 2 Batch 200 Loss 2.5285 Accuracy 0.1106\n",
      "Epoch 2 Batch 300 Loss 2.4815 Accuracy 0.1145\n",
      "Epoch 2 Batch 400 Loss 2.4513 Accuracy 0.1181\n",
      "Epoch 2 Batch 500 Loss 2.4194 Accuracy 0.1210\n",
      "Epoch 2 Batch 600 Loss 2.3962 Accuracy 0.1236\n",
      "Epoch 2 Batch 700 Loss 2.3721 Accuracy 0.1260\n",
      "Epoch 2 Loss 2.3713 Accuracy 0.1260\n",
      "Time taken for 1 epoch: 83.61259198188782 secs\n",
      "\n",
      "Epoch 3 Batch 0 Loss 2.3392 Accuracy 0.1427\n",
      "Epoch 3 Batch 100 Loss 2.1646 Accuracy 0.1451\n",
      "Epoch 3 Batch 200 Loss 2.1446 Accuracy 0.1452\n",
      "Epoch 3 Batch 300 Loss 2.1355 Accuracy 0.1462\n",
      "Epoch 3 Batch 400 Loss 2.1233 Accuracy 0.1480\n",
      "Epoch 3 Batch 500 Loss 2.1150 Accuracy 0.1497\n",
      "Epoch 3 Batch 600 Loss 2.1011 Accuracy 0.1516\n",
      "Epoch 3 Batch 700 Loss 2.0881 Accuracy 0.1535\n",
      "Epoch 3 Loss 2.0880 Accuracy 0.1536\n",
      "Time taken for 1 epoch: 84.19668126106262 secs\n",
      "\n",
      "Epoch 4 Batch 0 Loss 1.9075 Accuracy 0.1683\n",
      "Epoch 4 Batch 100 Loss 1.9249 Accuracy 0.1725\n",
      "Epoch 4 Batch 200 Loss 1.9139 Accuracy 0.1727\n",
      "Epoch 4 Batch 300 Loss 1.8970 Accuracy 0.1750\n",
      "Epoch 4 Batch 400 Loss 1.8824 Accuracy 0.1771\n",
      "Epoch 4 Batch 500 Loss 1.8659 Accuracy 0.1788\n",
      "Epoch 4 Batch 600 Loss 1.8539 Accuracy 0.1807\n",
      "Epoch 4 Batch 700 Loss 1.8441 Accuracy 0.1828\n",
      "Epoch 4 Loss 1.8437 Accuracy 0.1828\n",
      "Time taken for 1 epoch: 84.03117537498474 secs\n",
      "\n",
      "Epoch 5 Batch 0 Loss 1.5299 Accuracy 0.1867\n",
      "Epoch 5 Batch 100 Loss 1.6641 Accuracy 0.2011\n",
      "Epoch 5 Batch 200 Loss 1.6676 Accuracy 0.2025\n",
      "Epoch 5 Batch 300 Loss 1.6584 Accuracy 0.2036\n",
      "Epoch 5 Batch 400 Loss 1.6497 Accuracy 0.2048\n",
      "Epoch 5 Batch 500 Loss 1.6403 Accuracy 0.2059\n",
      "Epoch 5 Batch 600 Loss 1.6331 Accuracy 0.2070\n",
      "Epoch 5 Batch 700 Loss 1.6278 Accuracy 0.2084\n",
      "Saving checkpoint for epoch 5 at /content/drive/My Drive/Colab Notebooks/transformer/ckpt-1\n",
      "Epoch 5 Loss 1.6283 Accuracy 0.2085\n",
      "Time taken for 1 epoch: 85.19561910629272 secs\n",
      "\n",
      "Epoch 6 Batch 0 Loss 1.4623 Accuracy 0.2297\n",
      "Epoch 6 Batch 100 Loss 1.4823 Accuracy 0.2238\n",
      "Epoch 6 Batch 200 Loss 1.4701 Accuracy 0.2235\n",
      "Epoch 6 Batch 300 Loss 1.4676 Accuracy 0.2247\n",
      "Epoch 6 Batch 400 Loss 1.4632 Accuracy 0.2258\n",
      "Epoch 6 Batch 500 Loss 1.4570 Accuracy 0.2263\n",
      "Epoch 6 Batch 600 Loss 1.4522 Accuracy 0.2271\n",
      "Epoch 6 Batch 700 Loss 1.4439 Accuracy 0.2281\n",
      "Epoch 6 Loss 1.4435 Accuracy 0.2281\n",
      "Time taken for 1 epoch: 85.99724531173706 secs\n",
      "\n",
      "Epoch 7 Batch 0 Loss 1.2713 Accuracy 0.2430\n",
      "Epoch 7 Batch 100 Loss 1.2860 Accuracy 0.2443\n",
      "Epoch 7 Batch 200 Loss 1.2951 Accuracy 0.2459\n",
      "Epoch 7 Batch 300 Loss 1.2906 Accuracy 0.2469\n",
      "Epoch 7 Batch 400 Loss 1.2810 Accuracy 0.2477\n",
      "Epoch 7 Batch 500 Loss 1.2771 Accuracy 0.2482\n",
      "Epoch 7 Batch 600 Loss 1.2740 Accuracy 0.2487\n",
      "Epoch 7 Batch 700 Loss 1.2660 Accuracy 0.2492\n",
      "Epoch 7 Loss 1.2657 Accuracy 0.2492\n",
      "Time taken for 1 epoch: 85.94596433639526 secs\n",
      "\n",
      "Epoch 8 Batch 0 Loss 1.1447 Accuracy 0.2524\n",
      "Epoch 8 Batch 100 Loss 1.1042 Accuracy 0.2635\n",
      "Epoch 8 Batch 200 Loss 1.1116 Accuracy 0.2647\n",
      "Epoch 8 Batch 300 Loss 1.1162 Accuracy 0.2643\n",
      "Epoch 8 Batch 400 Loss 1.1156 Accuracy 0.2651\n",
      "Epoch 8 Batch 500 Loss 1.1157 Accuracy 0.2659\n",
      "Epoch 8 Batch 600 Loss 1.1162 Accuracy 0.2662\n",
      "Epoch 8 Batch 700 Loss 1.1157 Accuracy 0.2665\n",
      "Epoch 8 Loss 1.1163 Accuracy 0.2666\n",
      "Time taken for 1 epoch: 87.58165216445923 secs\n",
      "\n",
      "Epoch 9 Batch 0 Loss 0.8514 Accuracy 0.3019\n",
      "Epoch 9 Batch 100 Loss 0.9911 Accuracy 0.2827\n",
      "Epoch 9 Batch 200 Loss 0.9955 Accuracy 0.2808\n",
      "Epoch 9 Batch 300 Loss 1.0059 Accuracy 0.2818\n",
      "Epoch 9 Batch 400 Loss 1.0070 Accuracy 0.2815\n",
      "Epoch 9 Batch 500 Loss 1.0072 Accuracy 0.2814\n",
      "Epoch 9 Batch 600 Loss 1.0100 Accuracy 0.2811\n",
      "Epoch 9 Batch 700 Loss 1.0112 Accuracy 0.2808\n",
      "Epoch 9 Loss 1.0117 Accuracy 0.2808\n",
      "Time taken for 1 epoch: 87.4095528125763 secs\n",
      "\n",
      "Epoch 10 Batch 0 Loss 1.0691 Accuracy 0.2939\n",
      "Epoch 10 Batch 100 Loss 0.8989 Accuracy 0.2905\n",
      "Epoch 10 Batch 200 Loss 0.9048 Accuracy 0.2909\n",
      "Epoch 10 Batch 300 Loss 0.9113 Accuracy 0.2904\n",
      "Epoch 10 Batch 400 Loss 0.9154 Accuracy 0.2905\n",
      "Epoch 10 Batch 500 Loss 0.9202 Accuracy 0.2911\n",
      "Epoch 10 Batch 600 Loss 0.9232 Accuracy 0.2908\n",
      "Epoch 10 Batch 700 Loss 0.9274 Accuracy 0.2910\n",
      "Saving checkpoint for epoch 10 at /content/drive/My Drive/Colab Notebooks/transformer/ckpt-2\n",
      "Epoch 10 Loss 0.9275 Accuracy 0.2909\n",
      "Time taken for 1 epoch: 87.70754671096802 secs\n",
      "\n",
      "Epoch 11 Batch 0 Loss 0.8716 Accuracy 0.3028\n",
      "Epoch 11 Batch 100 Loss 0.8232 Accuracy 0.3009\n",
      "Epoch 11 Batch 200 Loss 0.8326 Accuracy 0.3014\n",
      "Epoch 11 Batch 300 Loss 0.8420 Accuracy 0.3007\n",
      "Epoch 11 Batch 400 Loss 0.8467 Accuracy 0.3004\n",
      "Epoch 11 Batch 500 Loss 0.8537 Accuracy 0.3010\n",
      "Epoch 11 Batch 600 Loss 0.8577 Accuracy 0.3001\n",
      "Epoch 11 Batch 700 Loss 0.8606 Accuracy 0.2995\n",
      "Epoch 11 Loss 0.8611 Accuracy 0.2995\n",
      "Time taken for 1 epoch: 86.9194552898407 secs\n",
      "\n",
      "Epoch 12 Batch 0 Loss 0.7313 Accuracy 0.3029\n",
      "Epoch 12 Batch 100 Loss 0.7818 Accuracy 0.3110\n",
      "Epoch 12 Batch 200 Loss 0.7859 Accuracy 0.3098\n",
      "Epoch 12 Batch 300 Loss 0.7890 Accuracy 0.3091\n",
      "Epoch 12 Batch 400 Loss 0.7948 Accuracy 0.3089\n",
      "Epoch 12 Batch 500 Loss 0.7989 Accuracy 0.3081\n",
      "Epoch 12 Batch 600 Loss 0.8039 Accuracy 0.3077\n",
      "Epoch 12 Batch 700 Loss 0.8075 Accuracy 0.3073\n",
      "Epoch 12 Loss 0.8078 Accuracy 0.3073\n",
      "Time taken for 1 epoch: 86.03745102882385 secs\n",
      "\n",
      "Epoch 13 Batch 0 Loss 0.7015 Accuracy 0.3045\n",
      "Epoch 13 Batch 100 Loss 0.7318 Accuracy 0.3197\n",
      "Epoch 13 Batch 200 Loss 0.7376 Accuracy 0.3187\n",
      "Epoch 13 Batch 300 Loss 0.7422 Accuracy 0.3174\n",
      "Epoch 13 Batch 400 Loss 0.7475 Accuracy 0.3159\n",
      "Epoch 13 Batch 500 Loss 0.7497 Accuracy 0.3153\n",
      "Epoch 13 Batch 600 Loss 0.7568 Accuracy 0.3154\n",
      "Epoch 13 Batch 700 Loss 0.7615 Accuracy 0.3142\n",
      "Epoch 13 Loss 0.7611 Accuracy 0.3142\n",
      "Time taken for 1 epoch: 86.3356282711029 secs\n",
      "\n",
      "Epoch 14 Batch 0 Loss 0.6588 Accuracy 0.3146\n",
      "Epoch 14 Batch 100 Loss 0.6804 Accuracy 0.3223\n",
      "Epoch 14 Batch 200 Loss 0.6888 Accuracy 0.3218\n",
      "Epoch 14 Batch 300 Loss 0.6982 Accuracy 0.3209\n",
      "Epoch 14 Batch 400 Loss 0.7071 Accuracy 0.3209\n",
      "Epoch 14 Batch 500 Loss 0.7114 Accuracy 0.3204\n",
      "Epoch 14 Batch 600 Loss 0.7157 Accuracy 0.3195\n",
      "Epoch 14 Batch 700 Loss 0.7201 Accuracy 0.3188\n",
      "Epoch 14 Loss 0.7200 Accuracy 0.3188\n",
      "Time taken for 1 epoch: 86.5377869606018 secs\n",
      "\n",
      "Epoch 15 Batch 0 Loss 0.6337 Accuracy 0.3730\n",
      "Epoch 15 Batch 100 Loss 0.6401 Accuracy 0.3297\n",
      "Epoch 15 Batch 200 Loss 0.6548 Accuracy 0.3280\n",
      "Epoch 15 Batch 300 Loss 0.6631 Accuracy 0.3268\n",
      "Epoch 15 Batch 400 Loss 0.6693 Accuracy 0.3264\n",
      "Epoch 15 Batch 500 Loss 0.6729 Accuracy 0.3256\n",
      "Epoch 15 Batch 600 Loss 0.6774 Accuracy 0.3245\n",
      "Epoch 15 Batch 700 Loss 0.6831 Accuracy 0.3238\n",
      "Saving checkpoint for epoch 15 at /content/drive/My Drive/Colab Notebooks/transformer/ckpt-3\n",
      "Epoch 15 Loss 0.6833 Accuracy 0.3239\n",
      "Time taken for 1 epoch: 87.99038195610046 secs\n",
      "\n",
      "Epoch 16 Batch 0 Loss 0.5221 Accuracy 0.3413\n",
      "Epoch 16 Batch 100 Loss 0.6207 Accuracy 0.3359\n",
      "Epoch 16 Batch 200 Loss 0.6226 Accuracy 0.3323\n",
      "Epoch 16 Batch 300 Loss 0.6311 Accuracy 0.3314\n",
      "Epoch 16 Batch 400 Loss 0.6369 Accuracy 0.3308\n",
      "Epoch 16 Batch 500 Loss 0.6403 Accuracy 0.3294\n",
      "Epoch 16 Batch 600 Loss 0.6474 Accuracy 0.3292\n",
      "Epoch 16 Batch 700 Loss 0.6535 Accuracy 0.3285\n",
      "Epoch 16 Loss 0.6534 Accuracy 0.3285\n",
      "Time taken for 1 epoch: 88.45781326293945 secs\n",
      "\n",
      "Epoch 17 Batch 0 Loss 0.6035 Accuracy 0.3351\n",
      "Epoch 17 Batch 100 Loss 0.5810 Accuracy 0.3387\n",
      "Epoch 17 Batch 200 Loss 0.5946 Accuracy 0.3372\n",
      "Epoch 17 Batch 300 Loss 0.6019 Accuracy 0.3356\n",
      "Epoch 17 Batch 400 Loss 0.6060 Accuracy 0.3347\n",
      "Epoch 17 Batch 500 Loss 0.6128 Accuracy 0.3343\n",
      "Epoch 17 Batch 600 Loss 0.6184 Accuracy 0.3336\n",
      "Epoch 17 Batch 700 Loss 0.6237 Accuracy 0.3328\n",
      "Epoch 17 Loss 0.6241 Accuracy 0.3328\n",
      "Time taken for 1 epoch: 87.9146032333374 secs\n",
      "\n",
      "Epoch 18 Batch 0 Loss 0.5749 Accuracy 0.3618\n",
      "Epoch 18 Batch 100 Loss 0.5620 Accuracy 0.3440\n",
      "Epoch 18 Batch 200 Loss 0.5683 Accuracy 0.3421\n",
      "Epoch 18 Batch 300 Loss 0.5790 Accuracy 0.3422\n",
      "Epoch 18 Batch 400 Loss 0.5846 Accuracy 0.3403\n",
      "Epoch 18 Batch 500 Loss 0.5892 Accuracy 0.3391\n",
      "Epoch 18 Batch 600 Loss 0.5937 Accuracy 0.3382\n",
      "Epoch 18 Batch 700 Loss 0.5990 Accuracy 0.3372\n",
      "Epoch 18 Loss 0.5994 Accuracy 0.3372\n",
      "Time taken for 1 epoch: 87.47308993339539 secs\n",
      "\n",
      "Epoch 19 Batch 0 Loss 0.5440 Accuracy 0.3380\n",
      "Epoch 19 Batch 100 Loss 0.5432 Accuracy 0.3466\n",
      "Epoch 19 Batch 200 Loss 0.5482 Accuracy 0.3428\n",
      "Epoch 19 Batch 300 Loss 0.5559 Accuracy 0.3430\n",
      "Epoch 19 Batch 400 Loss 0.5602 Accuracy 0.3428\n",
      "Epoch 19 Batch 500 Loss 0.5658 Accuracy 0.3420\n",
      "Epoch 19 Batch 600 Loss 0.5725 Accuracy 0.3412\n",
      "Epoch 19 Batch 700 Loss 0.5765 Accuracy 0.3405\n",
      "Epoch 19 Loss 0.5768 Accuracy 0.3405\n",
      "Time taken for 1 epoch: 86.16471123695374 secs\n",
      "\n",
      "Epoch 20 Batch 0 Loss 0.5388 Accuracy 0.3664\n",
      "Epoch 20 Batch 100 Loss 0.5112 Accuracy 0.3502\n",
      "Epoch 20 Batch 200 Loss 0.5247 Accuracy 0.3485\n",
      "Epoch 20 Batch 300 Loss 0.5323 Accuracy 0.3481\n",
      "Epoch 20 Batch 400 Loss 0.5370 Accuracy 0.3466\n",
      "Epoch 20 Batch 500 Loss 0.5432 Accuracy 0.3459\n",
      "Epoch 20 Batch 600 Loss 0.5477 Accuracy 0.3442\n",
      "Epoch 20 Batch 700 Loss 0.5551 Accuracy 0.3436\n",
      "Saving checkpoint for epoch 20 at /content/drive/My Drive/Colab Notebooks/transformer/ckpt-4\n",
      "Epoch 20 Loss 0.5552 Accuracy 0.3437\n",
      "Time taken for 1 epoch: 86.0255196094513 secs\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 训练：遍历数据集\n",
    "epochs = 20\n",
    "for epoch in range(epochs):\n",
    "    start = time.time()\n",
    "    train_loss.reset_states()\n",
    "    train_accuracy.reset_states()\n",
    "    \n",
    "    # 训练\n",
    "    for (batch, (inp, tar)) in enumerate(train_dataset):\n",
    "        train_step(inp, tar)\n",
    "        if batch % 100 == 0:\n",
    "            print('Epoch {} Batch {} Loss {:.4f} Accuracy {:.4f}'.format(\n",
    "                epoch + 1, batch, train_loss.result(), train_accuracy.result()))\n",
    "    \n",
    "    # 保存\n",
    "    if(epoch + 1) % 5 == 0:\n",
    "        ckpt_save_path = ckpt_manager.save()\n",
    "        print('Saving checkpoint for epoch {} at {}'.format(epoch + 1, ckpt_save_path))\n",
    "    \n",
    "    # 打印日志\n",
    "    print('Epoch {} Loss {:.4f} Accuracy {:.4f}'.format(\n",
    "        epoch + 1, train_loss.result(), train_accuracy.result()))\n",
    "    print('Time taken for 1 epoch: {} secs\\n'.format(time.time() - start))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nIOaPNn0W4Rb"
   },
   "source": [
    "## Evaluate and Visualize\n",
    "### Evaluate：teacher force\n",
    "\n",
    "eg: A B C D -> E F G H\n",
    "\n",
    "Train: A B C D, E F G -> F G H\n",
    "\n",
    "Eval: \n",
    "- A B C D -> E\n",
    "- A B C D, E -> F\n",
    "- A B C D, E F -> G\n",
    "- A B C D, E F G -> H"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pVbl_LxHW4Rb"
   },
   "outputs": [],
   "source": [
    "def evaluate(inp_sentence):\n",
    "    start_token = [pt_tokenizer.vocab_size]\n",
    "    end_token = [pt_tokenizer.vocab_size + 1]\n",
    "    # 文本转id\n",
    "    input_id_sentence = start_token + pt_tokenizer.encode(inp_sentence) + end_token\n",
    "    # 扩维\n",
    "    # encoder_input.shape: (1, input_seq_len)\n",
    "    encoder_input = tf.expand_dims(input_id_sentence, 0)\n",
    "    # decoder_input.shape: (1, 1)\n",
    "    decoder_input = tf.expand_dims([en_tokenizer.vocab_size], 0)\n",
    "    \n",
    "    for i in range(max_length):\n",
    "        # 创建mask\n",
    "        encoder_padding_mask, decoder_mask, encoder_decoder_padding_mask = create_masks(\n",
    "            encoder_input, decoder_input)\n",
    "        \n",
    "        # 预测\n",
    "        # predictions.shape: (batch_size, output_target_len, target_vocab_size)\n",
    "        predictions, attention_weights = transformer(\n",
    "            encoder_input, decoder_input, False, encoder_padding_mask,\n",
    "            decoder_mask, encoder_decoder_padding_mask)\n",
    "        # 取出预测序列的最后一个\n",
    "        # predictions.shape: (batch_size, target_vocab_size)\n",
    "        predictions = predictions[:, -1, :]\n",
    "        # 取最大值\n",
    "        predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)\n",
    "        \n",
    "        # 判断是否是最后一位\n",
    "        if tf.equal(predicted_id, en_tokenizer.vocab_size + 1):\n",
    "            # 因为之前扩维，故把第0个维度缩减\n",
    "            return tf.squeeze(decoder_input, axis = 0), attention_weights\n",
    "        \n",
    "        decoder_input = tf.concat([decoder_input, [predicted_id]], axis = -1)\n",
    "    return tf.squeeze(decoder_input, axis = 0), attention_weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "STvx_oz8W4Rd"
   },
   "source": [
    "### Visualize\n",
    "可视化Encoder和Decoder之间的attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sIOfz3BjW4Rd"
   },
   "outputs": [],
   "source": [
    "def plot_encoder_decoder_attention(attention, input_sentence, result, layer_name):\n",
    "    fig = plt.figure(figsize=(16, 8))\n",
    "    input_id_sentence = pt_tokenizer.encode(input_sentence)\n",
    "    \n",
    "    # attention[layer_name].shape: (batc_size=1, num_heads, tar_len, input_len)\n",
    "    # attention.shape: (num_heads, tar_len, input_len)\n",
    "    attention = tf.squeeze(attention[layer_name], axis = 0)\n",
    "    \n",
    "    # 画num_heads个子图\n",
    "    for head in range(attention.shape[0]):\n",
    "        ax = fig.add_subplot(2, 4, head + 1)\n",
    "        # 画矩阵,去掉最后一个\n",
    "        ax.matshow(attention[head][:-1, :])\n",
    "        # 设置字体\n",
    "        fontdict = {'fontsize': 10}\n",
    "        \n",
    "        # 设置锚点\n",
    "        ax.set_xticks(range(len(input_id_sentence) + 2))\n",
    "        ax.set_yticks(range(len(result)))\n",
    "        ax.set_ylim(len(result) - 1.5, -0.5)\n",
    "        \n",
    "        # 设置label\n",
    "        ax.set_xticklabels(\n",
    "            ['<start>'] + [pt_tokenizer.decode([i]) for i in input_id_sentence] + ['<end>'],\n",
    "            fontdict = fontdict, rotation = 90)\n",
    "        ax.set_yticklabels(\n",
    "            [en_tokenizer.decode([i]) for i in result if i < en_tokenizer.vocab_size],\n",
    "            fontdict = fontdict)\n",
    "        ax.set_xlabel('Head {}'.format(head + 1))\n",
    "        \n",
    "    # 自适应调整子图位置、间距\n",
    "    plt.tight_layout()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "dBb8jsliW4Rf"
   },
   "source": [
    "### Inference:Translate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "a6dM7Ae2W4Rf"
   },
   "outputs": [],
   "source": [
    "def translate(input_sentence, layer_name = ''):\n",
    "    result, attention_weights = evaluate(input_sentence)\n",
    "    predicted_sentence = en_tokenizer.decode(\n",
    "        [i for i in result if i < en_tokenizer.vocab_size])\n",
    "    \n",
    "    print('Input: {}'.format(input_sentence))\n",
    "    print('Predicted translation: {}'.format(predicted_sentence))\n",
    "    \n",
    "    if layer_name:\n",
    "        plot_encoder_decoder_attention(attention_weights, input_sentence, result, layer_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 53
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2769,
     "status": "ok",
     "timestamp": 1581688962752,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "8wSggUOaW4Ri",
    "outputId": "9e424e06-160c-4d95-9d28-e7c7a556121c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: está muito frio aqui\n",
      "Predicted translation: it 's very cold , it 's very cold .\n"
     ]
    }
   ],
   "source": [
    "translate('está muito frio aqui')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 53
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2159,
     "status": "ok",
     "timestamp": 1581688968628,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "t2xWA1W4W4Rj",
    "outputId": "bc781ae5-056b-47cd-a472-a04b1435ec87"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: istá muito frio aqui\n",
      "Predicted translation: it goes a lot of cold to this point here .\n"
     ]
    }
   ],
   "source": [
    "translate('istá muito frio aqui')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 53
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1500,
     "status": "ok",
     "timestamp": 1581688973322,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "5CIWxe6mW4Rl",
    "outputId": "71b506ff-3e55-4e63-aee5-3b61f31a2bb4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: você ainda está em casa?\n",
      "Predicted translation: are you still at home ?\n"
     ]
    }
   ],
   "source": [
    "translate('você ainda está em casa?')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 53
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1704,
     "status": "ok",
     "timestamp": 1581688976977,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "W2CWz_lyW4Rm",
    "outputId": "af224a78-f450-48a8-90a9-89c96115ec5b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: este é o primeiro livro que eu fiz.\n",
      "Predicted translation: this is the first book i did .\n"
     ]
    }
   ],
   "source": [
    "translate(\"este é o primeiro livro que eu fiz.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 583
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2753,
     "status": "ok",
     "timestamp": 1581688986239,
     "user": {
      "displayName": "Awebone Xu",
      "photoUrl": "",
      "userId": "14125540201237715804"
     },
     "user_tz": -480
    },
    "id": "MWkUwoi-W4Rq",
    "outputId": "1708ca81-10cc-4199-b57b-f1f13cf5f9f8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: este é o primeiro livro que eu fiz.\n",
      "Predicted translation: this is the first book i did .\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABHgAAAISCAYAAAC3TXhFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzde5RkdXnv//czd2BmuCOK4aqCoiA4\nR0XRg1eMMRe8HCOu/OItGI2XaDgeSYzGRHNMPDErmigiCBr9mRCVE4xBE4wCAmK4zoAwxJ+XeMEk\nAso4MLee5/dH7Z6uaXqG6q79rd3f6vdrrVrdtbv62U/vrvrM7mf23hWZiSRJkiRJkuq1qOsGJEmS\nJEmSNBwHPJIkSZIkSZVzwCNJkiRJklQ5BzySJEmSJEmVc8AjSZIkSZJUOQc8kiRJkiRJlXPAI0mS\nJEmSVDkHPJIkSZIkSZVzwCNJkiRJklQ5BzySJEmSJEmVW9J1A5r/IuJ44CnN3Ssy86Yu+5E03swc\nSaNk5kgaNXNHpXgEj3YrIt4IfBI4qLl9IiJe321XksaVmSNplMwcSaNm7qikyMyue9A8FhFrgZMy\nc2Nzfy/g6sw8rtvOJI0jM0fSKJk5kkbN3FFJHsGjBxLARN/9iWaZJJVg5kgaJTNH0qiZOypmbK7B\nExEBXASclZm3dt3PGDkfuCYiLmru/wpwXof9SPOGuVOEmSPtgplThJkj7YKZU4y5o2LG5hStiDgV\n+CjwN5n5O133M04i4kTg5ObuFZl5Q5f9SPOFuVOGmSPNzMwpw8yRZmbmlGPuqJRxGvBcSG8a+hfA\nozJzW8ctVS8iFgO3ZOYxXfcizUfmTrvMHGn3zJx2mTnS7pk57TN3VNpYXIMnIg4Ajs3MS4BL6R3m\npiFl5gSwPiIO7boXab4xd9pn5ki7Zua0z8yRds3MKcPcUWljMeABfg34VPP5+cCrOuxl3OwL3BIR\nX4qIiydvXTelbkXEaRGxsus+OmbulGHm6H7MHMDMKcXM0f2YOYCZU5K5o/tpK3fG4hStiFgHPCcz\nf9Dcvwl4XmZ+r9vO6hcR/32m5Zl52ah70fwQEUcBtwGvz8yzu+6nK+ZOGWaOpjNzesycMswcTWfm\n9Jg55Zg7mq7N3Kl+wBMR+wAvzswP9y17FvBjL1YltS8i3tV8+uzMfHynzXTE3JFGx8wxc6RRMnPM\nHGnU2syd6k/RysyfADdPW/bPwJ7ddDQeIuKrzccNEXFP321DRNzTdX/qRnNhuBcBfwL8NCKO77il\nTpg77TNzNBMzp8fMaZ+Zo5mYOT1mThnmjmbSdu5UP+BpfGDAZRpQZp7cfFyVmav7bqsyc3XX/akz\nzwW+lpkb6L1t5is77qdL5k6LzBztgpkzxcxpkZmjXTBzppg5LTN3tAut5s6SVlrqSEScBDwJODAi\n3tz3pdXA4m66Gj8RcTLw8Mw8v7mi/qrM/HbXfakTrwTe13x+EfCuiDgzM7d02NNImTvlmTnqY+aY\nOcWZOepj5pg5I2HuqE+ruVP7ETzLgJX0BlWr+m73AC/ssK+xERHvAP4XcFazaBnwie46Ulea87H3\nyczLATJzE/Bp4OmdNjZ65k5BZo4mmTk7mDkFmTmaZObsYOYUZu5oUoncGYeLLC8GLszMF3TdyziK\niBuBE4DrM/OEZtnazDyu286k7pg75Zg50v2ZOeWYOdL9mTllmTsqqepTtAAycyIiHtJ1H2NsS2Zm\nRCRAROzVdUMavYg4cXdfz8zrR9XLfGDuFGXmyMyZxswpysyRmTONmVOcuaNiuVP9gKdxY0RcDPwd\nsHFyYWZ+truWxsaFEfFhYJ+I+A3gFcBHOu5Jo/dnzccVwBrgJiCA44BrgZM66qtL5k4ZZo7AzJmJ\nmVOGmSMwc2Zi5pRj7ggK5U71p2gBRMT5MyzOzHzFyJsZQxHxLODZ9J5wX2zeJlELUER8FnhHZq5r\n7j8a+IPMXHDnZJs75Zg5mmTmTDFzyjFzNMnMmWLmlGXuaFLbuTMWAx6VFxGr6TviKzPv6rAddSQi\nbsnMYx9omTQsM0dg5mh0zByBmaPRMncE7efOWJyiFREr6L292LH0DnECwAnz8CLi1cA7gU3AdnpT\n5gSObKH2MuARzd31mbl12Joqbm1EnMvUlf5fCqztsJ/OmDtlmDmaxsxpmDllmDmaxsxpmDnllMod\nM6dareZO7W+TPumvgYOBU4HLgIcCG4YpGBEPiojzIuKS5v6jIuKVQ3danzOBR2fm4Zl5ZGYekZlt\n7PScAvwb8FfAB4HbI+Kpw9ZVcS8HbgHe2Ny+0SxbiFrNHTNnBzNH/cycKe7rlGHmqJ+ZM8XMKaf1\n3DFzqtZq7ozFKVoRcUNmnjD59nIRsRS4IjOfOETNS4Dzgd/LzOMjYglwQ2Y+pq2+axARXwCen5n3\ntlz3OuD0zFzf3H8E8KnMfFyb65FKaTt3zJweM0eamfs6ZZg50szMnHJK5I6Zo0ljcYoWMHn42U+a\nixL9CDhoyJoHZOaFEXEWQGZui4iJIWvW6Czgqoi4Btg8uTAz3zBk3aWTAdTUu735h2MoEXEY8PDM\nvDQi9gCWZOZQ/9ugKRHxZOAPgMPY+Zzhof+3s0Jt546Z01NV5oC5U5KZsxP3dcowc7SDmbMTM6ec\nErlj5lSq7dwZlwHPORGxL/A24GJgJfD7Q9bcGBH70zsfkoh4IvDTIWvW6MPAvwDr6J0j2pZrZzjX\n8NphCkbvbQbPAPYDjqJ3KOnZwDOGqaudnAe8CbgOWIj/IPdrO3fMnJ5qMgfMnREwc6a4r1OGmaN+\nZs4UM6ecErlj5tSr1dwZl1O0jsjMbz/QslnWPBH4APBo4GbgQOBFmXnTUM1WZvLwzAJ1lwO/BZzc\nLLoC+GBmbt71dz1gzRuBxwPXTPYcEesW2mGfJUXENZn5hK77mA/azh0zp6emzGnqmjsFmTlT3Ncp\nw8xRPzNniplTToncMXPq1XbujMuA5/rMPHHasuuGOeeweZFMAEfTu7L5emDRsC+S2kTEHwPfAT7H\nzocQzvlt/CJiMfDxzHzp0A3uXPeazHxC3znDS4DrM/O4NtezkEXEe4DFwGfZ+flwfWdNdaTt3DFz\nemrKnKa2uVOQmTPFfZ0yzBz1M3OmmDnltJ07Zk7d2s6dqk/Riohj6L11394R8fy+L62m7+385ujq\nJtRu6Vvf9cCJu/6WsfSS5uNZfcuGehu/zJyIiMMiYllmbhmqu51dFhG/C+wREc8CXksvONWeyeny\nmr5lCTy9g146UTB3zJyemjIHzJ3SzBz3dUozc9TPzDFzRqHV3DFzqtdq7lQ94KE3/X0esA/wi33L\nNwC/MZeCEXEwcAi9J/EJ9KbL0Au1Pefeap0y84hCpb8FXBkRFwMb+9b3viFqvhV4Jb3zWV8N/CNw\n7jBNameZ+bSue5gHWs0dM2dnlWUOmDtFmTmA+zpFmTnqZ+YAZk5xhXLHzKlU27kzLqdonZSZV7dU\n69eBl9GboP0rUwG0AbggMz/bxnrmu4h4emb+y7TJ/Q7DboeIeMcu6r5zmLoqKyIeBPwx8JDM/PmI\neBRwUmae13FrI9dW7pg5PWaOZmLmTHFfp11mjmZi5kwxc9pXMnfMnHq1nTvjMuD5U+BdwH3AF4Dj\ngDdl5id2+427r/mCzPxMSy1WJyLemZnviIjzZ/hyZuYrhqx/YtvnM0fEt2muyt8vh3xry2YbzFR3\nqG1Qo4i4BDgf+L3MPL45D/eGhXihtbZzx8ypL3OauuZOQWbOFPd12mXm3K+umYOZ08/MaV/J3DFz\n6tV27tR+itakZ2fmWyLiNHoXrHo+cDlTbxM3Fw+NiNX0JssfoXdu6Fsz85+GbbYGTfgsAi7JzAsL\nrOLPmsM1Pw38bWbe3ELN/vMWVwAvoveWfsP6h2l1TwN+OGzRiHgKcFVmTvQtKxLOLTogMy+MiLMA\nMnNbRCzUtxFtO3fMnPoyB8yd0sycKe7rtMjMuR8zp8fMmWLmtKxw7pg5VJk50HbuZGb1N+CW5uO5\nwHOaz28asuZNzcdTgYvoXWzs+q5/1g627bUFax8MvAG4kt55nW8rsI7rCtRcRC84hq1zL3AZcFDf\nsnn9HAO+Auw/2SfwROCyrvvqaFu0mjtmzo7tUHXmNOsxd9r7uc2cqW3hvk6Z7WrmzFzTzEkzp/lo\n5rS/bYvkjplTX+Y0/bWaO+NyBM/nIuI2eocQviYiDgQ2DVlz8tzQ59J727lbIiJ29w1j6tKIOBP4\nW3a+YNec3z60r8aPgPdHxJeBtwBvp3co6JxERP8V+BfRmziXeI4/HDiohTrrgffSuzr9KzPzKqae\nd/PVm4GLgaMi4krgQOCF3bbUmbZzx8zpqSZzwNwZATNnivs6ZZg5MzNzzBwzp5wiuWPmAPVlDrSc\nO2NxDR6AiNgP+Gn23iZuL2BV8ySfa73z6V3t/QjgeHrvTf+VzHxcKw1XojnvcrrM4c+7fCTwYuAF\nwJ30Au4zmfmfQ9T8ct/dbfQOJ/0/mbl+iFaJiA30zhGN5uOPgLNyyHOII+L6zDwxIh5O7+f/KPCK\n7L195LzVnBd6NL3tsT4zt3bcUmfazB0zp6emzGnqmjuFmTlT3Ndpn5mzo66Z0zBzppg5ZZTIHTNn\nR93qMgfazZ3qBzwRsSfw8My8qW/ZocBEZv5giLqLgMcC38rMn0TE/sAhmbl26KZFRFxN70V3YWYO\nfb5ljSLihsw8ofl8Jb0Aen5mzssj60q91mpUYluYOWWZOT015Y6ZM8V9nfqYOT1mTp3MnPqYOT01\nZQ4U+ptiDAY8S4HbgOMyc2Oz7J+A383Ma4eoG8BLgSMz8w+bDX1wZn59yH6L1G1qHw88pbl7Rf8T\nZYiaK4DXAifTm65eAZydmcMeotm6iHjz7r6eme+bY93J39kRmflHbf7OZljXoZn5723XbUOp11qN\nSmwLM2dHzWoyB8ydksycKe7r7Khr5pg5xZg5U8ycnWov6Nwxc8oq8Vpb1GJ/nWgOX7oI+B+wY+J1\nYAtB/EHgJOAlzf0NwF/NpVBEnBwRi9uuO20dbwQ+Se/cxYOAT0TE64etC3yc3gXQPgD8ZfP5Xw/R\n54XNx3URsbbvti4ihp3erwFeQ+/Qz0OA36R3df5VzW2uJn9npzf3h/qdRcRbmo8fiIj399+AM4fo\ns6iCr7XqFNoWZk5PTZkD5k4xZs4U93XMnD5mTiFmzhQzZ8c65n3umDk9NWYOFHqt5Ty4cvSwN+AY\n4PLm87cBb2ih5uRVrG/oWzanK8cDTwLOabvutHWsBfbqu78XsLaFut8YZNks6j24+XjYTLche72c\n3rnBk/dXTT4v5stzofneO5uPvw38+vTbsP2WvJV4rdV6a3tbmDk76lSTOU1dc6fgzcwpuy1qyh0z\nZ0d9M6fgzcwpuy1qypymxrzPHTNnx/dWmTlNz62+1ubluWizlZm3Rc8jgF9l6jC6YWxtpsIJEL0r\nx2+fY39XRcS9bdedJoCJvvsTzbJhXR8RT8zMrwFExBOAOU8UM/OO5uN3W+htugcBW/rub2mWDavt\n39l/RMRDgJcDp9DO72kkCr3WqlRgW5g5PTVlDpg7RZk5U9zXMXMaZk5BZs4UMweoIHfMnB2qzBxo\n/7U2FgOexnnAucC6zLy7hXrvp3e41EER8W56b1X2trkWy8wbS9Ttcz5wTURc1Nz/FXrbZFiPA66K\niMnzFg8F1kfEOnpXez9uNsVi6orp9/tSU2/1EL1+HPj6tG1wwRD1JrX9O/sQ8CXgSOC6vuWTV5Ef\n6p07diUiDs4h3vmgT9uvtZq1uS3MnJ6aMgfMnV0yc4pYyPs6Zk6PmbMbLeWOmTNlIWcOVJA7Zs4O\nNWcOtPhaq/4iy5OidwXqO4AXZOalLdU8BngGvSfGlzLz1nle90R6F+uC3kXAbmih5mG7+3rBafGc\nNNtgcup5eRvboKnb+u8sIj6Uma8ZurnB1/f5zPyFFuq0/lqrVdvbwsypL3PA3NnNusycli30fR0z\np8fM2e36hs4dM2fKQs+cpu6Czx0zZ7frm3f7OmMz4JEkSZIkSVqoqn8XLUmSJEmSpIXOAY8kSZIk\nSVLlxm7AExFnWNe6pWpat3zdGvn8tW6NdWvqtWTdGtW2jX3+WrfGumbOlJp+b9YtV9O6ddQduwEP\nUCqMrVtf3Zp6tW7dfP5at8a6NfVasm6NatvGPn+tW2NdM2dKTb8365arad0K6o7jgEeSJEmSJGlB\nqeJdtJbFitwj9hrosVvYzDKWD/TYTQ/dc+AeJn62kcUrB+thxY82D1x3y/b7WLZoj4Eem9smBq67\nNTexNFYM9NiJ/QbfDts2bWTJigfeDkvuvm/gmgBbchPLBug3t28fuOZWNrN0wOfCbFh39nU3cPeP\nM/PA1psoaNmiFbnH4lUDPXbL9k0sWzTY623/Y+4d6HEb7trGqv2WDPRYgDtvHex1POhrbVJODJY7\n8+F5Zt3ua86XupvYyJbcHK03UdCypXvlihX7DPTYrVs3snTpYPsksWnLwD3MJsuYxe7jlryPZTHY\nvg4x2K9tNvtPAJsfNNhzZ2LjRhbvNdi2XX7X4PtlW7ZtZNmSwermpk0D192am1kag77eBn9JzGY/\nkln8LTEf8mHQ59hstsGm3MiW3FRX5sTyXMFgz8nZbN8YcPvC7P5uO+jYwfafAH561wR777d4oMf+\nx82D58istsPyZQPX3TJxL8sWP/B+XG4ePM/nxWvNukXr7urvq8H/eujQHrEXT1zx3Nbrrv+fj229\nJsAj3/OdInUn7rq7SN27funE1mvu/5mbW68JsP1nPytSdzY7J7Myi3/kxtml2//uu133MFt7LF7F\nSfu+oPW6L7vo2tZrAlzwhPZfxwATP72nSF1y8GHt4DXn/39YaAYFcvKa7Ze2XrO0FSv24fGPfW3r\ndZfc9u+t1wRgwOHvrC0d/I+i2fjm6x/Wes2jPlVmvyy/8c0idVk82B+8s5VbBv+jcz6IZe0/x762\n+ZLWa5a2gr14Qjyj9bqLVgz+n0iz8ca/v6FI3T9/2COL1F3y0MNbr7nt24V2p91/qtKl+ekZnxCe\noiVJkiRJklQ5BzySJEmSJEmVc8AjSZIkSZJUOQc8kiRJkiRJlXPAI0mSJEmSVLk5DXgiYp+IeG3f\n/VMi4h928dhzI+JRc21QkswcSaNk5kgaNXNHUhvmegTPPsBA7+WZma/KzG/McT2SBGaOpNEycySN\nmrkjaWhzHfC8BzgqIm6MiPc2y1ZGxKcj4raI+GREBEBEfCUi1kTE4oi4ICJujoh1EfGmVn4CSQuB\nmSNplMwcSaNm7kga2pI5ft9bgUdn5mOhdwghcAJwLPBD4ErgycBX+77nscAhmfno5nv22d0KIuIM\n4AyAFbHXHNuUNCaKZ07zmKncWbSyxfYlVWb0mbN87xbbl1Sh0f59xZ4tty9pPmjzIstfz8zvZ+Z2\n4Ebg8Glf/xZwZER8ICKeA9yzu2KZeU5mrsnMNctY3mKbksZEq5kD03Jn0Yr2O5ZUs6KZs3Sp/5kl\n6X6K/X211L+vpLHU5oBnc9/nE0w7Oigz7waOB74C/CZwbovrlrTwmDmSRsnMkTRq5o6kWZnrKVob\ngFWz+YaIOADYkpmfiYj1wCfmuG5JC4+ZI2mUzBxJo2buSBranAY8mXlnRFwZETcDlwCfH+DbDgHO\nj4jJo4bOmsu6JS08Zo6kUTJzJI2auSOpDXM9gofMPH3aoq/0fe11fZ+f0veYE+e6PkkLm5kjaZTM\nHEmjZu5IGlab1+CRJEmSJElSBxzwSJIkSZIkVc4BjyRJkiRJUuUc8EiSJEmSJFXOAY8kSZIkSVLl\n5vwuWiO1aBGxYnnrZY8+7yet1wRg2dIiZRcfsH+Runc+c1PrNQ+4ZI/WawKwYUOZuqVkdt2B5mrR\nImKvPVsv+7HnPK31mgB3nP6QInUf/De3Fal796mPaL3mfld8v/WaABM/+s8idXNiokhdcnuhuoXy\nzJwEIDZtZcntP2i97ubHHtF6TYDl3/9pkbrcUeb19nP/vKX1mvcdsqr1mgDL15XJhihSFRY/qv08\nB5i49ZtF6uaW9p8L5tiURYX+Xvm5JYX+bitk27e+03ULWqA8gkeSJEmSJKlyDngkSZIkSZIq54BH\nkiRJkiSpcg54JEmSJEmSKueAR5IkSZIkqXIOeCRJkiRJkirngEeSJEmSJKlyxQc8EXFV6XVI0iQz\nR9KomTuSRsnMkbQrxQc8mfmk0uuQpElmjqRRM3ckjZKZI2lXRnEEz8+ajw+OiMsj4saIuDkinlJ6\n3ZIWHjNH0qiZO5JGycyRtCtLRriu04EvZua7I2IxsOfuHhwRZwBnAKxYtNcI2pM0ZmaVOTAtdxav\nKtyepDE0xL7OyhG0J2nMzD1zHni3SFKFRjng+VfgoxGxFPi/mXnj7h6cmecA5wDsveTAHEF/ksbL\nrDIHpuXO8geZO5Jma+77OksPMnMkzdacM2d17GfmSGNoZO+ilZmXA08FfgBcEBH/z6jWLWnhMXMk\njZq5I2mUzBxJ041swBMRhwH/kZkfAc4FThzVuiUtPGaOpFEzdySNkpkjabpRnqJ1CvA/I2Ir8DPA\nCbOkkk7BzJE0Wqdg7kganVMwcyT1KT7gycyVzcePAR8rvT5JC5uZI2nUzB1Jo2TmSNqVkZ2iJUmS\nJEmSpDIc8EiSJEmSJFXOAY8kSZIkSVLlHPBIkiRJkiRVzgGPJEmSJElS5Ub5NulzlhMTTPzkp+0X\n/uk97dcEcsnSMnUnJorU/YeTP916zTdveGbrNaVRyq1bmfjBHe3X3bat9ZoAB33oe0Xq3v35w4vU\nXXr29tZr5n33tV4TILdtLVKXzDJ1VaWcmGD7Pe3vlyz5l+tbrwkwEWX+j3Dx/vsVqfudV7SfOftf\nuqz1mgDLC2VDqX9/Jm5ZX6Su6rTt+z8oUvfMY55WpC6Lyvwb/85vfr31mn/05F9ovSbAtjt+VKSu\nuuERPJIkSZIkSZVzwCNJkiRJklQ5BzySJEmSJEmVc8AjSZIkSZJUOQc8kiRJkiRJlXPAI0mSJEmS\nVDkHPJIkSZIkSZUbasATEftExGv77p8SEf8wfFuSdH9mjqRRMnMkjZq5I2kYwx7Bsw/w2gd8lCS1\nw8yRNEpmjqRRM3ckzdmwA573AEdFxI0R8d5m2cqI+HRE3BYRn4yIAIiIx0XEZRFxXUR8MSIePOS6\nJS08Zo6kUTJzJI2auSNpzpYM+f1vBR6dmY+F3iGEwAnAscAPgSuBJ0fENcAHgF/OzP+KiBcD7wZe\nsavCEXEGcAbACvYcsk1JY6JY5jT1zB1J/cwcSaPm31eS5mzYAc9Mvp6Z3weIiBuBw4GfAI8G/rkZ\nOC8G7thdkcw8BzgHYHXslwX6lDQeWskcmJY7i8wdSTMqlDn7mzmSdsW/ryQNpMSAZ3Pf5xPNOgK4\nJTNPKrA+SQubmSNplMwcSaNm7kgayLDX4NkArBrgceuBAyPiJICIWBoRxw65bkkLj5kjaZTMHEmj\nZu5ImrOhBjyZeSdwZUTc3HcRsJketwV4IfAnEXETcCPwpGHWLWnhMXMkjZKZI2nUzB1Jwxj6FK3M\nPH3aoq/0fe11fZ/fCDx12PVJWtjMHEmjZOZIGjVzR9JcDXuKliRJkiRJkjrmgEeSJEmSJKlyDngk\nSZIkSZIq54BHkiRJkiSpcg54JEmSJEmSKjf0u2hVLbNM2a1bitQt5ZHL9my95vZ77229pjRSCblt\nW9ddDG77RJGyq3/+/ytS94s//EzrNU99yGNbrymNTCa5eXPXXQwuy2TOxI9/XKTuN572j63X/OWX\nndR6TYAye6dS3bZv2VqocJkse+KKxa3X3Paj/2i9psaPR/BIkiRJkiRVzgGPJEmSJElS5RzwSJIk\nSZIkVc4BjyRJkiRJUuUc8EiSJEmSJFXOAY8kSZIkSVLlHPBIkiRJkiRVbtYDnoh4Q0TcGhGfjIhf\nioi3zuJ7D4+I02e7TkkLm7kjaZTMHEmjZOZIasuSOXzPa4FnZub3m/sXT39ARCzJzG0zfO/hwOnA\n/zuH9UpauMwdSaNk5kgaJTNHUitmNeCJiLOBI4FLIuKjwN3Amsx8XURcAGwCTgCujIi/B/6i+dYE\nngq8B3hkRNwIfCwz/7ydH0PSuDJ3JI2SmSNplMwcSW2a1YAnM38zIp4DPC0zfxwRL5v2kIcCT8rM\niYj4HPBbmXllRKykF05vBc7MzOc90Loi4gzgDIAV7DmbNiWNEXNH0iiZOZJGycyR1Ka2L7L8d5k5\n0Xx+JfC+iHgDsM8uDincpcw8JzPXZOaapSxvuU1JY8TckTRKZo6kUTJzJA2s7QHPxslPMvM9wKuA\nPegdUnhMy+uSJDB3JI2WmSNplMwcSQOby0WWBxIRR2XmOmBdRPw34Bjge8CqUuuUtLCZO5JGycyR\nNEpmjqQH0vYRPP1+OyJujoi1wFbgEmAtMBERN0XEmwquW9LCZO5IGiUzR9IomTmSdmvWR/Bk5uF9\nn18AXNB8/rJpj3v9Lko8fbbrlLSwmTuSRsnMkTRKZo6ktpQ8gkeSJEmSJEkj4IBHkiRJkiSpcg54\nJEmSJEmSKueAR5IkSZIkqXLF3iZ9QYsoU3bJ0iJ1n3vMU1uv+b7vfLH1mgBvPvykInUlzWzxvvsW\nqfsLJ57aes3TvnFz6zUB/n7NYUXqbt+0uUhdtk+UqStV7JePPLn1mp/69pdbrwnwq4c+uUhdMsvU\nlUZg0R4ritTdfu+9ReqeesgJrdf85L9/tfWaAC/9uUKZo054BI8kSZIkSVLlHPBIkiRJkiRVzgGP\nJEmSJElS5RzwSJIkSZIkVc4BjyRJkiRJUuUc8EiSJEmSJFXOAY8kSZIkSVLlHPBIkiRJkiRVbrcD\nnog4PCJubmNFEfGdiDigjVqSxpOZI2nUzB1Jo2TmSCrJI3gkSZIkSZIqN8iAZ0lEfDIibo2IT0fE\nngAR8YyIuCEi1kXERyNi+e6WT4qIPSLikoj4jQI/j6T6mTmSRs3ckTRKZo6kIgYZ8BwNfDAzHwnc\nA7w2IlYAFwAvzszHAEuA1+xqeV+tlcDngE9l5kd2t9KIOCMiro2Ia7eyeZY/lqSKdZI5YO5IC5j7\nOpJGycyRVMQgA57vZeaVzVYlbQwAACAASURBVOefAE6mF0rfzszbm+UfA566m+WT/h44PzM//kAr\nzcxzMnNNZq5ZyvIHerik8dFJ5oC5Iy1g7utIGiUzR1IRgwx48gHuz8aVwHMiIoaoIWm8mTmSRs3c\nkTRKZo6kIgYZ8BwaESc1n58OfBVYDxweEQ9rlv8acNlulk96O3A38FfDNi5pbJk5kkbN3JE0SmaO\npCIGGfCsB34rIm4F9gU+lJmbgJcDfxcR64DtwNm7Wj6t3huBPSLiT9v6ISSNFTNH0qiZO5JGycyR\nVMSS3X0xM78DHLOLr30JOGEWyw/vu/vy2TQpaWEwcySNmrkjaZTMHEklDXIEjyRJkiRJkuYxBzyS\nJEmSJEmVc8AjSZIkSZJUOQc8kiRJkiRJldvtRZbnjQhi+fLWy+bmza3XLGnxAft13cLAfu1P3lyk\n7pa3RpG6h559S5G62zfeV6Quub1M2W3bitSt1qLF7dfcPtF+zYIm7r676xYG9n+PP6RI3Xff/pUi\ndX//+GcWqTtxzz1F6hJl8pfMMnUrE4sXsXjl6tbrTmzY0HpNAKLQ/xE+/tgiZe86eq/Waz7vd+53\nzdtW7P3Iu4rUze/+oEzdyvYdcsuWAkXbL1lcBLF0Wetlc2uB7Qtsv29TkbqL99mnSN3Yc4/Wa572\nxjJ/X3FambIrflzmubD09jJZFsvafz0AbL+r0L70z2Ze7BE8kiRJkiRJlXPAI0mSJEmSVDkHPJIk\nSZIkSZVzwCNJkiRJklQ5BzySJEmSJEmVc8AjSZIkSZJUOQc8kiRJkiRJlXPAI0mSJEmSVLlOBjwR\ncVUX65W0cJk7kkbJzJE0SmaOJOhowJOZT+pivZIWLnNH0iiZOZJGycyRBN0dwfOzLtYraeEydySN\nkpkjaZTMHEkAS7puYFci4gzgDIAV7NlxN5IWAnNH0ijtlDmxV8fdSBp37udI42/eXmQ5M8/JzDWZ\nuWZprOi6HUkLwE65w/Ku25E05vozZ9ki93UkleXfV9L4m7cDHkmSJEmSJA3GAY8kSZIkSVLlHPBI\nkiRJkiRVrqu3SV/ZxXolLVzmjqRRMnMkjZKZIwk8gkeSJEmSJKl6DngkSZIkSZIq54BHkiRJkiSp\ncg54JEmSJEmSKrek6wYGkzAx0XUTg8ssUnbbHT8qUpeI1kse+OH/ar0mwOJ99y5Sd8sJRxWp+19v\nvK9I3UNefWeRutvv+kmRumwpU7a47RXljsitZZ5ob3/KaUXqxt7tZy/A7e89ukjdoz9SJs8W3be1\n9Zrxza+2XrO0nNjOxD33dN3G4LJQPn5tbZGy+36tSNkiavuX54s/vLFI3VMPOaFI3VL76dXJLPbv\nZhGF9skm7r67SF0K1N3zsz9svSZQ5G9BgFi8uEjdz33360XqPvehjytSd9SZ4xE8kiRJkiRJlXPA\nI0mSJEmSVDkHPJIkSZIkSZVzwCNJkiRJklQ5BzySJEmSJEmVc8AjSZIkSZJUOQc8kiRJkiRJlVtS\nsnhE/AHwM2A1cHlmXjrt66cAZ2bm80r2IWlhMHMkjZq5I2mUzBxJu1N0wDMpM98+ivVIEpg5kkbP\n3JE0SmaOpJm0fopWRPxeRNweEV8Fjm6WXRARL2w+f05E3BYR1wPPb3v9khYWM0fSqJk7kkbJzJE0\nqFYHPBHxOOBXgccCzwX+27SvrwA+Avwi8Djg4DbXL2lhMXMkjZq5I2mUzBxJs9H2ETxPAS7KzHsz\n8x7g4mlfPwb4dmb+W2Ym8IldFYqIMyLi2oi4dmtubrlNSWOitcyBabmDuSNpRmX2dcwcSTMzcyQN\nbN6+i1ZmnpOZazJzzdJY3nU7khaAnXIHc0dSWWaOpFEyc6Tx1/aA53LgVyJij4hYRe9QwX63AYdH\nxFHN/Ze0vH5JC4uZI2nUzB1Jo2TmSBpYq++ilZnXR8TfAjcB/wn867Svb4qIM4DPR8S9wBXAqjZ7\nkLRwmDmSRs3ckTRKZo6k2Wj9bdIz893Au3fz9S/QO1dUkoZm5kgaNXNH0iiZOZIGNW+vwSNJkiRJ\nkqTBOOCRJEmSJEmqnAMeSZIkSZKkyjngkSRJkiRJqpwDHkmSJEmSpMq1/i5aJcTixSzae3XrdSfu\nvKv1mlXKLFBzov2aQG7aXKTu8u/cWaTuQ1+3rUjdb7zrsCJ1H/axhxSpyxVlypYUe6xg0SPaf0OK\n7Wtva72mytr2/R903cKsPOLV3y9Sd8MXjixSd8937916zVxS3/9fxR4rWHT0I1uvu/2mW1uvWVRE\nmbol9nUK9RpLlhapy6Iy/T73+GcVqXv72YcXqXvEZ9p/LuTVV7des7gIYumy1svm1i2t11RZix9x\nVJG6E3vvUaTumnc/vkjdlaeV+ft11c0/LlKX9TMvrm8PSJIkSZIkSTtxwCNJkiRJklQ5BzySJEmS\nJEmVc8AjSZIkSZJUOQc8kiRJkiRJlXPAI0mSJEmSVDkHPJIkSZIkSZVzwCNJkiRJklQ5BzySJEmS\nJEmVc8AjSZIkSZJUuSVdN7ArEXEGcAbAikUrO+5G0kKwU+4sXd1xN5LG3c6Zs3fH3UgadztlDnt2\n3I2kEubtETyZeU5mrsnMNcsWrei6HUkLwE65s2SvrtuRNOZ2zhz/2JJUVn/mLA3/vpLG0bwd8EiS\nJEmSJGkwDngkSZIkSZIq1/mAJyLOjYg1XfchaWEwcySNmrkjaZTMHGnh6vwiy5n5qq57kLRwmDmS\nRs3ckTRKZo60cHV+BI8kSZIkSZKG44BHkiRJkiSpcg54JEmSJEmSKueAR5IkSZIkqXIOeCRJkiRJ\nkioXmdl1Dw8oIv4L+O6ADz8A+HGBNqxbX92aeh33uodl5oEFeihmHuTOfPi9WXe869bU62zrmjlz\nU1Pdmnq1bp11zZwpNf3erFuupnXnV90Zc6eKAc9sRMS1mbnGutatqVfr1s3nr3VrrFtTryXr1qi2\nbezz17o11jVzptT0e7NuuZrWraOup2hJkiRJkiRVzgGPJEmSJElS5cZxwHOOda1bsKZ1y9etkc9f\n69ZYt6ZeS9atUW3b2OevdWusa+ZMqen3Zt1yNa1bQd2xuwaPRi8ifpaZK/vuvwxYk5mva6H2V4Az\nM/PaactfB/w2cBRwYGaWuMiVpHmoo8z5JLAG2Ap8HXh1Zm4ddn2S5r+OMuc8epkTwO3AyzLzZ8Ou\nT1Idusidvq+/H3hF//pVj3E8gkcLw5XAMxn86v+SNIxPAscAjwH2AF7VbTuSxtybMvP4zDwO+Hdg\n6D/qJOmBRMQaYN+u+9DcOeBRURFxYER8JiL+tbk9uVn++Ii4OiJuiIirIuLoZvkeEfE3EXFrRFxE\n7w+p+8nMGzLzO6P7SSTVoGDm/GM26B3B89CR/VCS5q2CmXNP8/hoHuMh95KAcrkTEYuB9wJvGdkP\no9Yt6boBjYU9IuLGvvv7ARc3n/8F8OeZ+dWIOBT4IvBI4DbgKZm5LSKeCfwx8ALgNcC9mfnIiDgO\nuH5kP4WkWnSWORGxFPg14I2t/kSS5rNOMicizgeeC3wD+J22fyhJ81oXufM64OLMvKM3W1aNHPCo\nDfdl5mMn70yeI9rcfSbwqL6QWB0RK4G9gY9FxMPp/a/U0ubrTwXeD5CZayNibfn2JVWmy8z5IHB5\nZl7Rxg8iqQqdZE5mvrz5H/UPAC8Gzm/tJ5I03400dyLiIcCLgFNa/0k0Ug54VNoi4ImZual/YUT8\nJfDlzDwtIg4HvjL61iSNoWKZExHvAA4EXj18m5LGRNH9nMyciIi/oXfKhAMeSVAmd04AHgZ8sxkc\n7RkR38zMh7XSsUbGa/CotH8CXj95JyImJ9F7Az9oPn9Z3+MvB05vHvto4LjyLUoaI0UyJyJeBZwK\nvCQzt7fbsqSKtZ450fOwyc+BX6J36oUkQYHcyczPZ+bBmXl4Zh5O75QuhzsVcsCj0t4ArImItRHx\nDeA3m+V/CvzviLiBnY8k+xCwMiJuBf4QuG6mohHxhoj4Pr0Lna6NiHOL/QSSalIkc4CzgQcBV0fE\njRHx9jLtS6pMicwJeqdZrAPWAQ9uHitJUG5fR2Mgem8IIkmSJEmSpFp5BI8kSZIkSVLlHPBIkiRJ\nkiRVzgGPJEmSJElS5RzwSJIkSZIkVc4BjyRJkiRJUuUc8EiSJEmSJFXOAY8kSZIkSVLlHPBIkiRJ\nkiRVzgGPJEmSJElS5RzwSJIkSZIkVc4BjyRJkiRJUuUc8EiSJEmSJFXOAY8kSZIkSVLlHPBIkiRJ\nkiRVzgGPJEmSJElS5RzwSJIkSZIkVc4BjyRJkiRJUuUc8EiSJEmSJFXOAY8kSZIkSVLlHPBIkiRJ\nkiRVzgGPJEmSJElS5RzwSJIkSZIkVc4BjyRJkiRJUuUc8EiSJEmSJFXOAY8kSZIkSVLlHPBIkiRJ\nkiRVzgGPJEmSJElS5RzwSJIkSZIkVc4BjyRJkiRJUuUc8EiSJEmSJFXOAY8kSZIkSVLlHPBIkiRJ\nkiRVzgGPJEmSJElS5RzwSJIkSZIkVc4BjyRJkiRJUuUc8EiSJEmSJFXOAY8kSZIkSVLlHPBIkiRJ\nkiRVzgGPJEmSJElS5RzwSJIkSZIkVW5J1w1o/ouI44GnNHevyMybuuxH0ngzcySNkpkjadTMHZXi\nETzarYh4I/BJ4KDm9omIeH23XUkaV2aOpFEycySNmrmjkiIzu+5B81hErAVOysyNzf29gKsz87hu\nO5M0jswcSaNk5kgaNXNHJXkEjx5IABN99yeaZZJUgpkjaZTMHEmjZu6omLG5Bk9EBHARcFZm3tp1\nP2PkfOCaiLiouf8rwHkd9iPNG+ZOEWaOtAtmThFmjrQLZk4x5o6KGZtTtCLiVOCjwN9k5u903c84\niYgTgZObu1dk5g1d9iPNF+ZOGWaONDMzpwwzR5qZmVOOuaNSxmnAcyG9aehfAI/KzG0dt1S9iFgM\n3JKZx3TdizQfmTvtMnOk3TNz2mXmSLtn5rTP3FFpY3ENnog4ADg2My8BLqV3mJuGlJkTwPqIOLTr\nXqT5xtxpn5kj7ZqZ0z4zR9o1M6cMc0eljcWAB/g14FPN5+cDr+qwl3GzL3BLRHwpIi6evHXdlLoV\nEadFxMqu++iYuVOGmaP7MXMAM6cUM0f3Y+YAZk5J5o7up63cGYtTtCJiHfCczPxBc/8m4HmZ+b1u\nO6tfRPz3mZZn5mWj7kXzQ0QcBdwGvD4zz+66n66YO2WYOZrOzOkxc8owczSdmdNj5pRj7mi6NnOn\n+gFPROwDvDgzP9y37FnAj71YldS+iHhX8+mzM/PxnTbTEXNHGh0zx8yRRsnMMXOkUWszd6o/RSsz\nfwLcPG3ZPwN7dtPReIiIrzYfN0TEPX23DRFxT9f9qRvNheFeBPwJ8NOIOL7jljph7rTPzNFMzJwe\nM6d9Zo5mYub0mDllmDuaSdu5U/2Ap/GBAZdpQJl5cvNxVWau7rutyszVXfenzjwX+FpmbqD3tpmv\n7LifLpk7LTJztAtmzhQzp0VmjnbBzJli5rTM3NEutJo7S1ppqSMRcRLwJODAiHhz35dWA4u76Wr8\nRMTJwMMz8/zmivqrMvPbXfelTrwSeF/z+UXAuyLizMzc0mFPI2XulGfmqI+ZY+YUZ+aoj5lj5oyE\nuaM+reZO7UfwLANW0htUreq73QO8sMO+xkZEvAP4X8BZzaJlwCe660hdac7H3iczLwfIzE3Ap4Gn\nd9rY6Jk7BZk5mmTm7GDmFGTmaJKZs4OZU5i5o0klcmccLrK8GLgwM1/QdS/jKCJuBE4Ars/ME5pl\nazPzuG47k7pj7pRj5kj3Z+aUY+ZI92fmlGXuqKSqT9ECyMyJiHhI132MsS2ZmRGRABGxV9cNafQi\n4sTdfT0zrx9VL/OBuVOUmSMzZxozpygzR2bONGZOceaOiuVO9QOexo0RcTHwd8DGyYWZ+dnuWhob\nF0bEh4F9IuI3gFcAH+m4J43enzUfVwBrgJuAAI4DrgVO6qivLpk7ZZg5AjNnJmZOGWaOwMyZiZlT\njrkjKJQ71Z+iBRAR58+wODPzFSNvZgxFxLOAZ9N7wn2xeZtELUAR8VngHZm5rrn/aOAPMnPBnZNt\n7pRj5miSmTPFzCnHzNEkM2eKmVOWuaNJbefOWAx4VF5ErKbviK/MvKvDdtSRiLglM499oGXSsMwc\ngZmj0TFzBGaORsvcEbSfO2NxilZErKD39mLH0jvECQAnzMOLiFcD7wQ2AdvpTZkTOLKF2suARzR3\n12fm1mFrqri1EXEuU1f6fymwtsN+OmPulGHmaBozp2HmlGHmaBozp2HmlFMqd8ycarWaO7W/Tfqk\nvwYOBk4FLgMeCmwYpmBEPCgizouIS5r7j4qIVw7daX3OBB6dmYdn5pGZeURmtrHTcwrwb8BfAR8E\nbo+Ipw5bV8W9HLgFeGNz+0azbCFqNXfMnB3MHPUzc6a4r1OGmaN+Zs4UM6ec1nPHzKlaq7kzFqdo\nRcQNmXnC5NvLRcRS4IrMfOIQNS8Bzgd+LzOPj4glwA2Z+Zi2+q5BRHwBeH5m3tty3euA0zNzfXP/\nEcCnMvNxba5HKqXt3DFzeswcaWbu65Rh5kgzM3PKKZE7Zo4mjcUpWsDk4Wc/aS5K9CPgoCFrHpCZ\nF0bEWQCZuS0iJoasWaOzgKsi4hpg8+TCzHzDkHWXTgZQU+/25h+OoUTEYcDDM/PSiNgDWJKZQ/1v\ng6ZExJOBPwAOY+dzhof+384KtZ07Zk5PVZkD5k5JZs5O3Ncpw8zRDmbOTsycckrkjplTqbZzZ1wG\nPOdExL7A24CLgZXA7w9Zc2NE7E/vfEgi4onAT4esWaMPA/8CrKN3jmhbrp3hXMNrhykYvbcZPAPY\nDziK3qGkZwPPGKaudnIe8CbgOmAh/oPcr+3cMXN6qskcMHdGwMyZ4r5OGWaO+pk5U8ycckrkjplT\nr1ZzZ1xO0ToiM7/9QMtmWfNE4APAo4GbgQOBF2XmTUM1W5nJwzML1F0O/BZwcrPoCuCDmbl519/1\ngDVvBB4PXDPZc0SsW2iHfZYUEddk5hO67mM+aDt3zJyemjKnqWvuFGTmTHFfpwwzR/3MnClmTjkl\ncsfMqVfbuTMuA57rM/PEacuuG+acw+ZFMgEcTe/K5uuBRcO+SGoTEX8MfAf4HDsfQjjnt/GLiMXA\nxzPzpUM3uHPdazLzCX3nDC8Brs/M49pcz0IWEe8BFgOfZefnw/WdNdWRtnPHzOmpKXOa2uZOQWbO\nFPd1yjBz1M/MmWLmlNN27pg5dWs7d6o+RSsijqH31n17R8Tz+760mr6385ujq5tQu6VvfdcDJ+76\nW8bSS5qPZ/UtG+pt/DJzIiIOi4hlmbllqO52dllE/C6wR0Q8C3gtveBUeyany2v6liXw9A566UTB\n3DFzemrKHDB3SjNz3NcpzcxRPzPHzBmFVnPHzKleq7lT9YCH3vT3ecA+wC/2Ld8A/MZcCkbEwcAh\n9J7EJ9CbLkMv1Pace6t1yswjCpX+FnBlRFwMbOxb3/uGqPlW4JX0zmd9NfCPwLnDNKmdZebTuu5h\nHmg1d8ycnVWWOWDuFGXmAO7rFGXmqJ+ZA5g5xRXKHTOnUm3nzriconVSZl7dUq1fB15Gb4L2r0wF\n0Abggsz8bBvrme8i4umZ+S/TJvc7DLsdIuIdu6j7zmHqqqyIeBDwx8BDMvPnI+JRwEmZeV7HrY1c\nW7lj5vSYOZqJmTPFfZ12mTmaiZkzxcxpX8ncMXPq1XbujMuA50+BdwH3AV8AjgPelJmf2O037r7m\nCzLzMy21WJ2IeGdmviMizp/hy5mZrxiy/oltn88cEd+muSp/vxzyrS2bbTBT3aG2QY0i4hLgfOD3\nMvP45jzcGxbihdbazh0zp77MaeqaOwWZOVPc12mXmXO/umYOZk4/M6d9JXPHzKlX27lT+ylak56d\nmW+JiNPoXbDq+cDlTL1N3Fw8NCJW05ssf4TeuaFvzcx/GrbZGjThswi4JDMvLLCKP2sO1/w08LeZ\neXMLNfvPW1wBvIjeW/oN6x+m1T0N+OGwRSPiKcBVmTnRt6xIOLfogMy8MCLOAsjMbRGxUN9GtO3c\nMXPqyxwwd0ozc6a4r9MiM+d+zJweM2eKmdOywrlj5lBl5kDbuZOZ1d+AW5qP5wLPaT6/aciaNzUf\nTwUuonexseu7/lk72LbXFqx9MPAG4Ep653W+rcA6ritQcxG94Bi2zr3AZcBBfcvm9XMM+Aqw/2Sf\nwBOBy7ruq6Nt0WrumDk7tkPVmdOsx9xp7+c2c6a2hfs6ZbarmTNzTTMnzZzmo5nT/rYtkjtmTn2Z\n0/TXau6MyxE8n4uI2+gdQviaiDgQ2DRkzclzQ59L723nbomI2N03jKlLI+JM4G/Z+YJdc3770L4a\nPwLeHxFfBt4CvJ3eoaBzEhH9V+BfRG/iXOI5/nDgoBbqrAfeS+/q9K/MzKuYet7NV28GLgaOiogr\ngQOBF3bbUmfazh0zp6eazAFzZwTMnCnu65Rh5szMzDFzzJxyiuSOmQPUlznQcu6MxTV4ACJiP+Cn\n2XubuL2AVc2TfK71zqd3tfcjgOPpvTf9VzLzca00XInmvMvpMoc/7/KRwIuBFwB30gu4z2Tmfw5R\n88t9d7fRO5z0/2Tm+iFaJSI20DtHNJqPPwLOyiHPIY6I6zPzxIh4OL2f/6PAK7L39pHzVnNe6NH0\ntsf6zNzacUudaTN3zJyemjKnqWvuFGbmTHFfp31mzo66Zk7DzJli5pRRInfMnB11q8scaDd3qh/w\nRMSewMMz86a+ZYcCE5n5gyHqLgIeC3wrM38SEfsDh2Tm2qGbFhFxNb0X3YWZOfT5ljWKiBsy84Tm\n85X0Auj5mTkvj6wr9VqrUYltYeaUZeb01JQ7Zs4U93XqY+b0mDl1MnPqY+b01JQ5UOhvijEY8CwF\nbgOOy8yNzbJ/An43M68dom4ALwWOzMw/bDb0wZn59SH7LVK3qX088JTm7hX9T5Qhaq4AXgucTG+6\negVwdmYOe4hm6yLizbv7ema+b451J39nR2TmH7X5O5thXYdm5r+3XbcNpV5rNSqxLcycHTWryRww\nd0oyc6a4r7Ojrplj5hRj5kwxc3aqvaBzx8wpq8RrbVGL/XWiOXzpIuB/wI6J14EtBPEHgZOAlzT3\nNwB/NZdCEXFyRCxuu+60dbwR+CS9cxcPAj4REa8fti7wcXoXQPsA8JfN5389RJ8XNh/XRcTavtu6\niBh2er8GeA29Qz8PAX6T3tX5VzW3uZr8nZ3e3B/qdxYRb2k+fiAi3t9/A84cos+iCr7WqlNoW5g5\nPTVlDpg7xZg5U9zXMXP6mDmFmDlTzJwd65j3uWPm9NSYOVDotZbz4MrRw96AY4DLm8/fBryhhZqT\nV7G+oW/ZnK4cDzwJOKftutPWsRbYq+/+XsDaFup+Y5Bls6j34ObjYTPdhuz1cnrnBk/eXzX5vJgv\nz4Xme+9sPv428OvTb8P2W/JW4rVW663tbWHm7KhTTeY0dc2dgjczp+y2qCl3zJwd9c2cgjczp+y2\nqClzmhrzPnfMnB3fW2XmND23+lqbl+eizVZm3hY9jwB+lanD6IaxtZkKJ0D0rhy/fY79XRUR97Zd\nd5oAJvruTzTLhnV9RDwxM78GEBFPAOY8UczMO5qP322ht+keBGzpu7+lWTastn9n/xERDwFeDpxC\nO7+nkSj0WqtSgW1h5vTUlDlg7hRl5kxxX8fMaZg5BZk5U8wcoILcMXN2qDJzoP3X2lgMeBrnAecC\n6zLz7hbqvZ/e4VIHRcS76b1V2dvmWiwzbyxRt8/5wDURcVFz/1fobZNhPQ64KiImz1s8FFgfEevo\nXe39uNkUi6krpt/vS0291UP0+nHg69O2wQVD1JvU9u/sQ8CXgCOB6/qWT15Ffqh37tiViDg4h3jn\ngz5tv9Zq1ua2MHN6asocMHd2ycwpYiHv65g5PWbObrSUO2bOlIWcOVBB7pg5O9ScOdDia636iyxP\nit4VqO8AXpCZl7ZU8xjgGfSeGF/KzFvned0T6V2sC3oXAbuhhZqH7e7rBafFc9Jsg8mp5+VtbIOm\nbuu/s4j4UGa+ZujmBl/f5zPzF1qo0/prrVZtbwszp77MAXNnN+syc1q20Pd1zJweM2e36xs6d8yc\nKQs9c5q6Cz53zJzdrm/e7euMzYBHkiRJkiRpoar+XbQkSZIkSZIWurEb8ETEGda1bqma1i1ft0Y+\nf61bY92aei1Zt0a1bWOfv9atsa6ZM6Wm35t1y9W0bh11x27AA5QKY+vWV7emXq1bN5+/1q2xbk29\nlqxbo9q2sc9f69ZY18yZUtPvzbrlalq3grrjOOCRJEmSJElaUKq4yPKyWJ4rYq+BHrs1N7M0lg/0\n2AOP3TxwD/fctY3V+w32rvL/dfNg6wfYymaWMvjjS9SNxYsHrrslN7EsVsy1rd3UvY9lsccDPi4n\nJgauOR+2rXV7NnD3jzPzwNabKGhZLM8VDJg7s9gWP/eYnw30uLvv2s6++w0+g//eupUDPW4+PB8A\nYunSgR63Zft9LFv0wNkAkFu3Drz++bIduqxbU6+zrbuJjWzJzdF6EwWVypyHPGbjwD385M7t7LP/\nYLlzx+37Dlx3y8R9LFs84Ot485aBHjcfnmfWHe+64585K2bx99Umlg66/7/n4H8nbN26kaVLB+zh\n4MH/Zt12z70sWb3nQI9d+q1NA9edzd+ZsWg2f18N9ncQs/i7fTZ/s/3/7d17sF1neR7w55V0ZNmW\nLzg2AYJB2IANuGCwIFwCMZcODqUtTtyBuJMWWtAQQ7h0YErSFpg0BhpmYAK0psYF00JpBiiBQAkN\nTAhgKGDAYIMxEBsSLiUE8A0jWTrn6x86KrKw5HNZ39pa5/x+Mx7vvc4+z/q2zj6P1n619t5tYWHJ\nucv6Xaul/0os5zFWm5bxZ7us48g9S849HLrsYM+vljaxmLEtdXQevumJg+c+6z1fGzwzSS4+7dQu\nuak+J1xtPO7Y4UPb+meaZQAAGJdJREFU0otiOeavv6FLLn19uL3rsPq4x6XYkqPzy/X4wXNf8/5P\nDZ6ZJP/qXo/skrucg4nl2PSLdxs8c893vjt4JtP06faRWS9h2Xp1zkvf9/nBM5Pkwiee1yV3/hvX\ndcnt1WWQTLRz6ug8fO6cwXPbA08fPDNJvv3ipf8j73Kc/NRruuRu2Lq0wdVyLGcAsRwLt9zSJbc2\nb+6Su/GEpf8Dw3Ls+f4PuuT2el784YV33u7zKy/RAgAAAJg4Ax4AAACAiTPgAQAAAJg4Ax4AAACA\niTPgAQAAAJg4Ax4AAACAiVvRgKeqjq+qC/a7fnZVvf8gt72kqu6/0gUC6BxgTDoHGJveAYaw0jN4\njk9ywR3eKklr7Zmtta+scD8Aic4BxqVzgLHpHWDVVjrgeVWSU6vqiqp69eK2rVX1rqr6alW9vaoq\nSarqo1W1vao2VtWlVXVVVV1ZVS8c5B4A64HOAcakc4Cx6R1g1Tat8PtekuSM1tqZyd5TCJM8OMkD\nknw3yWVJHpXkE/t9z5lJfqm1dsbi9xx/qB1U1Y4kO5JkS45a4TKBNaJ75yzeRu8Aic4Bxuf5FbBq\nQ77J8mdaa99urS0kuSLJtgO+fm2SU6rq9VV1TpIbDxXWWru4tba9tbZ9ro4YcJnAGjFo5yQH9E70\nDnAbOgcYW8fnV1v6rBiYqSEHPLv2uzyfA84Oaq39OMmDknw0ybOTXDLgvoH1R+cAY9I5wNj0DrAs\nK32J1k1JjlnON1TViUluba29u6quSfK2Fe4bWH90DjAmnQOMTe8Aq7aiAU9r7YdVdVlVXZXkg0k+\nsIRv+6Ukb6mqfWcN/e5K9g2sPzoHGJPOAcamd4AhrPQMnrTWzj9g00f3+9pz97t89n63echK9wes\nbzoHGJPOAcamd4DVGvI9eAAAAACYAQMeAAAAgIkz4AEAAACYOAMeAAAAgIkz4AEAAACYuBV/itZa\ncMl5T+qUfE2X1Jrr8+M69v01eOZNT906eGaS5Pob+uTCAWrTxmy80y8Mnvuip/zLwTOT5PvPPb5L\n7t3e0afPfvjYewye+Quf6NORe775111yU8N3b5KktT65U1vvxNSRW7LhvqcPnnvhI+48eGaS1Kad\nXXI33vmkLrnfO+/eg2fO3dznsXunt36qS24vNbe5S27bs7tLbhcTrLHauDEbjj9u8Nz25esGz0yS\nW244rUtube7z+N25ffjOOfLL3xk8M0kWbrmlS2679dYuuQs/vr5Lbm3oc5zT9oxbEM7gAQAAAJg4\nAx4AAACAiTPgAQAAAJg4Ax4AAACAiTPgAQAAAJg4Ax4AAACAiTPgAQAAAJi47gOeqvpk730A7KNz\ngLHpHWBMOgc4mO4DntbaI3vvA2AfnQOMTe8AY9I5wMGMcQbPzYv/v2tVfayqrqiqq6rq0b33Daw/\nOgcYm94BxqRzgIPZNOK+zk/yodbahVW1MclRh7pxVe1IsiNJthz6pgC3Z1mdkxzQOxu2dl4esAat\n/Fhn7tgRlgesMSvvHMc5sCaNOeD5bJI3V9Vckj9prV1xqBu31i5OcnGSHLvhhDbC+oC1ZVmdk9y2\nd46bO0nvAMu14mOd4466m84BlmvlnTN3Z50Da9Bon6LVWvtYksck+U6SS6vqn421b2D90TnA2PQO\nMCadAxxotAFPVd0zyfdba29KckmSh4y1b2D90TnA2PQOMCadAxxozJdonZ3kxVW1O8nNSUyYgZ7O\njs4BxnV29A4wnrOjc4D9dB/wtNa2Lv7/rUne2nt/wPqmc4Cx6R1gTDoHOJjRXqIFAAAAQB8GPAAA\nAAATZ8ADAAAAMHEGPAAAAAATZ8ADAAAAMHFjfkz6yrWkzc8Pn3v1tcNnJklrXWI33u0uXXI/c8Xw\nucc9ZePgmUly5zd8p0suHKjtmc/8D380eO7G6jNXv8tFX+uSu/vhZ3TJPfKff2/wzPbnOwfP7KrT\n3xXdTG29E9N+ujMLV3191suYuff89ae65J5794cNnllzmwfPTJKp/aa13bfOegmsSEsWhn9+VZvn\nBs9MktOf+5UuuRtOuFOX3BtOHb4fbjr5XoNnJskJl36/S+7GY47pkptOj7GaX+iSO3/99V1yD/aX\nhTN4AAAAACbOgAcAAABg4gx4AAAAACbOgAcAAABg4gx4AAAAACbOgAcAAABg4lY14Kmq46vqgv2u\nn11V71/9sgB+ns4BxqRzgLHpHWA1VnsGz/FJLrjDWwEMQ+cAY9I5wNj0DrBiqx3wvCrJqVV1RVW9\nenHb1qp6V1V9tareXlWVJFV1VlX9ZVV9rqo+VFV3XeW+gfVH5wBj0jnA2PQOsGKrHfC8JMlftdbO\nbK29eHHbg5O8IMn9k5yS5FFVNZfk9UnOa62dleTNSS5c5b6B9UfnAGPSOcDY9A6wYps6ZH6mtfbt\nJKmqK5JsS3J9kjOS/PniwHljku8dKqSqdiTZkSRbclSHZQJrxCCds/j9ege4IzoHGNvwz682bO24\nXGBWegx4du13eX5xH5Xky621Ryw1pLV2cZKLk+TYOqENukJgLRmkcxK9AyyJzgHGNvjzq+PmTtI5\nsAat9iVaNyU5Zgm3uybJSVX1iCSpqrmqesAq9w2sPzoHGJPOAcamd4AVW9WAp7X2wySXVdVV+70J\n2O3d7tYk5yX5D1X1xSRXJHnkavYNrD86BxiTzgHGpneA1Vj1S7Raa+cfsOmj+33tuftdviLJY1a7\nP2B90znAmHQOMDa9A6zUal+iBQAAAMCMGfAAAAAATJwBDwAAAMDEGfAAAAAATJwBDwAAAMDEGfAA\nAAAATNyqPyZ9NK0NH7n71sEze9pz3be65F577nsHz3zic84cPBNGVUlt3Dh47ML1Nwye2dPG/3NV\nl9zX3edjg2e+8EdnD54Jo1qYn/UKlq6qS+y5J/9yl9wPffcLg2eec8+HDZ4JY2l75jP/wx/Nehkz\nt3DLLV1yP/eyDwye+cS7nzV4ZpIuz7OTZP7GG7vkcmjO4AEAAACYOAMeAAAAgIkz4AEAAACYOAMe\nAAAAgIkz4AEAAACYOAMeAAAAgIlb9oCnqp5XVVdX1dur6h9V1UuW8b3bqur85e4TWN/0DjAmnQOM\nSecAQ9m0gu+5IMkTWmvfXrz+vgNvUFWbWmt7bud7tyU5P8l/X8F+gfVL7wBj0jnAmHQOMIhlDXiq\n6o1JTknywap6c5IfJ9neWntuVV2aZGeSBye5rKrem+SPFr+1JXlMklcluV9VXZHkra211w5zN4C1\nSu8AY9I5wJh0DjCkZQ14WmvPrqpzkjy2tfZ3VfX0A25y9ySPbK3NV9WfJnlOa+2yqtqaveX0kiQv\naq09+Y72VVU7kuxIki05ajnLBNYQvQOMSecAY9I5wJCGfpPld7bW5hcvX5bkNVX1vCTHH+SUwoNq\nrV3cWtveWts+lyMGXiawhvTpndI7wO1yrAOMSecASzb0gOcn+y601l6V5JlJjszeUwpPH3hfAIne\nAcalc4Ax6RxgyVbyJstLUlWnttauTHJlVT00yelJ/ibJMb32CaxvegcYk84BxqRzgDsy9Bk8+3tB\nVV1VVV9KsjvJB5N8Kcl8VX2xql7Ycd/A+qR3gDHpHGBMOgc4pGWfwdNa27bf5UuTXLp4+ekH3O53\nDhLxuOXuE1jf9A4wJp0DjEnnAEPpeQYPAAAAACMw4AEAAACYOAMeAAAAgIkz4AEAAACYOAMeAAAA\ngIlb9qdoMUNVXWLPucf2wTPf8+1PDp6ZJOfe/WFdcuHntKTt2TPrVcxep955wbZHDp75oe9+ZvDM\nJHni3c7skgu3UZWa2zx4bNt96+CZPW3YurVL7q/de/jOefk1Hx88M0ledspZXXKzYWOf3LbQJ7c6\n/Tv0wnyf3Cnq9Hd8F611ia1NfZ4O/9p9HjV45iv+6i8Gz0yS37tXp+dXU3p8JanNw/8dnCRt164u\nuQfjDB4AAACAiTPgAQAAAJg4Ax4AAACAiTPgAQAAAJg4Ax4AAACAiTPgAQAAAJg4Ax4AAACAiTvk\ngKeqtlXVVUPsqKq+WVUnDpEFrE06Bxib3gHGpHOAnpzBAwAAADBxSxnwbKqqt1fV1VX1rqo6Kkmq\n6vFV9YWqurKq3lxVRxxq+z5VdWRVfbCqntXh/gDTp3OAsekdYEw6B+hiKQOe05L8p9ba/ZLcmOSC\nqtqS5NIkT22t/b0km5L89sG275e1NcmfJnlHa+1Nh9ppVe2oqsur6vLd2bXMuwVM2Ew6J9E7sI7N\n/lin7Rz6PgGHr9l3juMcWJOWMuD5m9baZYuX35bkV7K3lK5rrX1tcftbkzzmENv3eW+St7TW/usd\n7bS1dnFrbXtrbftcjrijmwNrx0w6J9E7sI7N/lintgxxP4BpmH3nOM6BNWkpA552B9eX47Ik51RV\nrSIDWNt0DjA2vQOMSecAXSxlwHOPqnrE4uXzk3wiyTVJtlXVvRe3/1aSvzzE9n1emuTHSf7jahcO\nrFk6Bxib3gHGpHOALpYy4LkmyXOq6uokd0pyUWttZ5JnJHlnVV2ZZCHJGw+2/YC85yc5sqr+cKg7\nAawpOgcYm94BxqRzgC42HeqLrbVvJjn9IF/7SJIHL2P7tv2uPmM5iwTWB50DjE3vAGPSOUBPSzmD\nBwAAAIDDmAEPAAAAwMQZ8AAAAABMnAEPAAAAwMQZ8AAAAABM3CE/ReuwUjXrFSxda7NewbK0PXsG\nz3zMy58/eGaSnHT0l7rkfu2N9+2Se59/8eUuuWkLfWI7PBYmrUfvTKwfeq1355MfNnjmP/iVbYNn\nJsmGM7Z0yb32aXfqknvKK77YJbc2z3XJnb/hxuFDJ/ZrliRpLW33rbNexdJ16oaFm27qktvDy045\na9ZLWJbr3n5Gl9xNVx3dJffkCz/ZJZf9TO2YpINex749cn/v1EcMnpkkqT7PK6575cO75C7M9Xnc\n3uff9XmeOfZvmTN4AAAAACbOgAcAAABg4gx4AAAAACbOgAcAAABg4gx4AAAAACbOgAcAAABg4gx4\nAAAAACZuJgOeqvrkLPYLrF96BxiTzgHGpHOAZEYDntbaI2exX2D90jvAmHQOMCadAySzO4Pn5lns\nF1i/9A4wJp0DjEnnAEmyadYLOJiq2pFkR5JsyVEzXg2wHugdYEw6BxiTzoG177B9k+XW2sWtte2t\nte1zOWLWywHWAb0DjEnnAGPSObD2HbYDHgAAAACWxoAHAAAAYOIMeAAAAAAmblYfk751FvsF1i+9\nA4xJ5wBj0jlA4gweAAAAgMkz4AEAAACYOAMeAAAAgIkz4AEAAACYOAMeAAAAgInbNOsFLEVt3JCN\nW48ZPHf+xhsHz5yiDVu2DJ554n/7/OCZSbLhpBO75J72ipu75H71NQ/uknu/136/S25a65N7bZ/Y\n7nr9eZAtH/js4Jl7ev28qrrEnvIHR3TJfcM1H+6S+5z7Pr5Lbsq/NSVJqlJHDP+YaLt2DZ7Jog0b\n++S2hS6xR33q6C65N5yxu0vuhqP7rLc2bx4+84ZOj4WedM70dOqGTfc8uUvufV7/rS651/+X4Z+7\nJsmGXzypS25d32nm8KPb3+yoCgAAAGDiDHgAAAAAJs6ABwAAAGDiDHgAAAAAJs6ABwAAAGDiDHgA\nAAAAJs6ABwAAAGDiNvUMr6qXJ7k5ybFJPtZa+/ABXz87yYtaa0/uuQ5gfdA5wNj0DjAmnQMcStcB\nzz6ttZeOsR+AROcA49M7wJh0DnB7Bn+JVlX9m6r6WlV9Islpi9surarzFi+fU1VfrarPJ/n1ofcP\nrC86Bxib3gHGpHOApRp0wFNVZyV5WpIzkzwpyUMP+PqWJG9K8g+TnJXkLofI2lFVl1fV5bcu7Bxy\nmcAaMWTnLN7+//fO7uzqs2hg0nod6+xujnWAn6dzgOUY+gyeRyd5T2vtltbajUned8DXT09yXWvt\n6621luRtBwtqrV3cWtveWtu+ecOWgZcJrBGDdU5y296ZyxGdlgxMXJdjnblyrAPcLp0DLJlP0QIA\nAACYuKEHPB9L8pSqOrKqjsneUwX399Uk26rq1MXrvznw/oH1RecAY9M7wJh0DrBkg36KVmvt81X1\nx0m+mORvk3z2gK/vrKodST5QVbck+XiSY4ZcA7B+6BxgbHoHGJPOAZZj8I9Jb61dmOTCQ3z9z7L3\ntaIAq6ZzgLHpHWBMOgdYKu/BAwAAADBxBjwAAAAAE2fAAwAAADBxBjwAAAAAE2fAAwAAADBxg3+K\nVhdzc8nd7zJ87tU3DZ+ZJK1NKndh167hQzuttR19ZJfc+Wuu7ZJ72n/e2CX3G8+4a5fce7230+9E\nnz/evo7aknrAGYPHtsuvGjyTRVVdYjced2yX3IWf7uyS+5x7P65L7rX//iFdck+4avi/L+Y/cNng\nmd0dtSULDxr+Q3DqU18cPJNFC/NdYmtuc5fcu1x0eZfcu3U6NvveMx7UJXfX8R0yL/ng8KGd1RGb\ns2HbyYPnzl/zjcEzu+p07NBFr+dX19/QJXfhJz/tknvMkxe65P7gnz60S+4PH9zneVuef/ubncED\nAAAAMHEGPAAAAAATZ8ADAAAAMHEGPAAAAAATZ8ADAAAAMHEGPAAAAAATZ8ADAAAAMHEGPAAAAAAT\nZ8ADAAAAMHEGPAAAAAATd9gOeKpqR1VdXlWX3zp/y6yXA6wD+/fO7j16B+jrNp2z+yezXg6wxt3m\n+ZXjHFiTDtsBT2vt4tba9tba9s0bj5r1coB1YP/emdukd4C+btM5c0fPejnAGneb51eOc2BNOmwH\nPAAAAAAsjQEPAAAAwMTNfMBTVZdU1fZZrwNYH3QOMDa9A4xJ58D6tWnWC2itPXPWawDWD50DjE3v\nAGPSObB+zfwMHgAAAABWx4AHAAAAYOIMeAAAAAAmzoAHAAAAYOIMeAAAAAAmrlprs17DHaqqHyT5\n1hJvfmKSv+uwDLnTy53SWtd67j1bayd1WEM3h0HvHA4/N7lrO3dKa11urs5ZmSnlTmmtcqeZq3N+\nZko/N7n9MuUeXrm32zuTGPAsR1Vd3lrbLlfulNYqd9o8fuVOMXdKa+2ZO0VT+zP2+JU7xVyd8zNT\n+rnJ7Zcpdxq5XqIFAAAAMHEGPAAAAAATtxYHPBfLldsxU27/3Cny+JU7xdwprbVn7hRN7c/Y41fu\nFHN1zs9M6ecmt1+m3Ankrrn34GF8VXVza23rftefnmR7a+25A2R/NMmLWmuXH7D90iS/muSGxU1P\nb61dsdr9AYe/GXVOJfmDJP8kyXySi1prr1vt/oDD34w65+NJjlm8euckn2mtPWW1+wOmYUa98/gk\nr87ek0Buzt7nV99Y7f4Y16ZZLwBW4cWttXfNehHAuvD0JCcnOb21tlBVd57xeoA1rLX26H2Xq+rd\nSd47w+UA68NFSf5xa+3qqrogyb/N3uMfJmQtvkSLw0hVnVRV766qzy7+96jF7Q+rqk9V1Req6pNV\nddri9iOr6n9U1dVV9Z4kR870DgCT0rFzfjvJ77fWFpKktfa3o9wh4LDW+zinqo5N8rgkf9L9zgCT\n0LF3WpJjFy8fl+S73e8Mg3MGD0M4sqr2f3nUCUnet3j5j5K8trX2iaq6R5IPJblfkq8meXRrbU9V\nPSHJK5L8RvY+ibqltXa/qnpgks8fYr8XVtVLk3wkyUtaa7uGvVvAYWoWnXNqkqdW1blJfpDkea21\nrw9+z4DD0ayOc5LkKUk+0lq7ccD7Axz+ZtE7z0zyv6rqp0luTPLwwe8V3RnwMISfttbO3Hdl32tE\nF68+Icn99759RZLk2Kramr1T4bdW1X2yd1o8t/j1xyR5XZK01r5UVV86yD5/N8n/TbI5e9+M6l8n\n+f2h7hBwWJtF5xyRZGdrbXtV/XqSNyd59EFuC6wts+icfX4zySVD3AlgUmbROy9M8qTW2qer6sVJ\nXpO9Qx8mxICH3jYkeXhrbef+G6vqDUn+orV2blVtS/LR5YS21r63eHFXVb0lyYtWv1RgDejSOUm+\nneR/Ll5+T5K3rG6ZwBrRq3NSVScmeViSc1e/TGANGbx3quqkJA9qrX16cdMfJ/mzQVbLqLwHD739\n7yS/s+9KVe2bRB+X5DuLl5++3+0/luT8xduekeSBtxdaVXdd/H9l7+nLVw25aGCyunRO9r7/xWMX\nL/9qkq8Ns1xg4np1TpKcl+T9Bz6JA9a9Hr3z4yTHVdV9F6///SRXD7dkxmLAQ2/PS7K9qr5UVV9J\n8uzF7X+Y5JVV9YXc9kyyi5Jsraqrs/clV587SO7bq+rKJFcmOTF7P74YoFfnvCrJbyz2zivjlGVg\nr16dkyRPS/KODmsGpm3w3mmt7UnyrCTvrqovJvmtJC/ueB/opFprs14DAAAAAKvgDB4AAACAiTPg\nAQAAAJg4Ax4AAACAiTPgAQAAAJg4Ax4AAACAiTPgAQAAAJg4Ax4AAACAift/CrZMYNGtzhcAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1152x576 with 8 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "translate(\"este é o primeiro livro que eu fiz.\", layer_name='decoder_layer4_att2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "v3iWGMTWW4Rt"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "32.transformer_colab.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "00d1991457d541d48c1ca5aad7fe3e8f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "info",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_365509bb19e3492ca4cf0467a65677cf",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_9ebf269ea639433484a26cffb896048f",
      "value": 1
     }
    },
    "02fa22f70da849ceb5cef6c6a9a6c42f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "056d566b9dfc4c12b13b0fcb6075a270": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "06d9c755ed0e4758a602472ba8a7c886": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0e04ea39154b40819a6ae5b5914847a2": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e65cf9ab4221419e97f74426d0ee292f",
      "placeholder": "​",
      "style": "IPY_MODEL_21fec75ed3724763aa3cd93b6e2c7ab9",
      "value": " 68% 35326/51785 [00:00&lt;00:00, 87505.93 examples/s]"
     }
    },
    "1a1013a121c6452a9dcf5be7d91b6bb6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "1a6d255cf1744383b9d216d4a04f0acf": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "1f0d1a90ce324a4c946574e4dc1affba": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "2121f86cde9c4eefb6b465d07b9e01df": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "21fec75ed3724763aa3cd93b6e2c7ab9": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "24187cb2ff0e45f9833a8c48c96efcc7": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "245051a8a9e746568bc24399fded1713": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2966a7701dde4f46885083de9c6b1eb0": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_84890da093d9455b8bfc29a6f432bc8f",
      "placeholder": "​",
      "style": "IPY_MODEL_1a6d255cf1744383b9d216d4a04f0acf",
      "value": "1066 examples [00:00, 5043.56 examples/s]"
     }
    },
    "2b125be6a55f43798366013f4d4ee840": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "34171b17684e49708e0633eed1c8216c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_ed874e7a5d294e28940f3ac0dcc99a96",
       "IPY_MODEL_9bf6fd00b1b64af2ba4b1574f0e66bc0"
      ],
      "layout": "IPY_MODEL_d8b58951df5c4dfeb6a1d24669b522ff"
     }
    },
    "365509bb19e3492ca4cf0467a65677cf": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "39e1c098df36444c932002921ed9bd51": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3dbe4b29ee9d467cbeef39946925db10": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4469976e9ed0411eb472ce953b2b942f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "46ec15c44cf0445eb7097b9ffcd48b72": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5599b46c0a374da4a1bc3566747f395c": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5d733f7bd7f245ce8e92cd7a74c9a967": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "danger",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b1f9c517f8404821aa7e374d674982e8",
      "max": 1803,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_7dc0f107d2be48de8be39dfbaff50c64",
      "value": 0
     }
    },
    "5ffbbf72f5dd402ca28cd2d35d79295b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "6276134a3fa7469faa2cdb9b9841a236": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Dl Size...",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b234ad53c4f048b3b336d9a3e5441189",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_1f0d1a90ce324a4c946574e4dc1affba",
      "value": 1
     }
    },
    "66234e0238854b9cb36cb57a1f2aa44f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_bb6f1ba6e0cf46889984cfeb9bd16097",
      "placeholder": "​",
      "style": "IPY_MODEL_5ffbbf72f5dd402ca28cd2d35d79295b",
      "value": "  0% 0/1803 [00:00&lt;?, ? examples/s]"
     }
    },
    "66669c75d14146139cba84f57a4c12ac": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "68c2817c35724d908b2dc27ddbc8d65a": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "76125a19d0e24b21a3e0cae93fe34b60": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_2121f86cde9c4eefb6b465d07b9e01df",
      "placeholder": "​",
      "style": "IPY_MODEL_80bca50374f8438bacd78207ad6c8e39",
      "value": "1159 examples [00:00, 5181.69 examples/s]"
     }
    },
    "792f241ea8394e9ca81eeac6909481f8": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7dc0f107d2be48de8be39dfbaff50c64": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "80bca50374f8438bacd78207ad6c8e39": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "84890da093d9455b8bfc29a6f432bc8f": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "84e50d0a4e824d409249d1c05f13a8fc": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "8543fc0a11d841bb9593d942eaf05601": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_eaa4b7905b524f5ea5e2d5c73437e0d5",
       "IPY_MODEL_76125a19d0e24b21a3e0cae93fe34b60"
      ],
      "layout": "IPY_MODEL_c003c5f9ade94009a41a9d225bc12502"
     }
    },
    "8636cf713e6548aca61d10c821199873": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_39e1c098df36444c932002921ed9bd51",
      "placeholder": "​",
      "style": "IPY_MODEL_89a784492dfb49a1a83aa693ef82a7b1",
      "value": "51742 examples [00:08, 7022.27 examples/s]"
     }
    },
    "89a784492dfb49a1a83aa693ef82a7b1": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "8a66272607e04f92b935580029abe3a1": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_5d733f7bd7f245ce8e92cd7a74c9a967",
       "IPY_MODEL_66234e0238854b9cb36cb57a1f2aa44f"
      ],
      "layout": "IPY_MODEL_d2077508138b42e688b069d59ba044a0"
     }
    },
    "8e4d74fcb67d49378495b45510ff668c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_00d1991457d541d48c1ca5aad7fe3e8f",
       "IPY_MODEL_8636cf713e6548aca61d10c821199873"
      ],
      "layout": "IPY_MODEL_985545ca832b43fbb75382850f1c39f1"
     }
    },
    "90a547d5b93746ceae81fd001be9004f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "918b595ca82f4e889e8851e7a3301542": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_db2ba1f0367d4dc4abcea8e19736a80f",
       "IPY_MODEL_e1050782a98a48c7ba72adacd240b84a"
      ],
      "layout": "IPY_MODEL_06d9c755ed0e4758a602472ba8a7c886"
     }
    },
    "985545ca832b43fbb75382850f1c39f1": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9bf6fd00b1b64af2ba4b1574f0e66bc0": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_24187cb2ff0e45f9833a8c48c96efcc7",
      "placeholder": "​",
      "style": "IPY_MODEL_90a547d5b93746ceae81fd001be9004f",
      "value": "  0% 0/1193 [00:00&lt;?, ? examples/s]"
     }
    },
    "9ebf269ea639433484a26cffb896048f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "a74e3d4851ad40a9802864c7e9fa3313": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e44536ead67141a58ce0aaad0ede93d2",
      "placeholder": "​",
      "style": "IPY_MODEL_1a1013a121c6452a9dcf5be7d91b6bb6",
      "value": "124/|/100% 124/124 [00:05&lt;00:00, 23.27 MiB/s]"
     }
    },
    "a845cf55d3cc451cbf6023f53451b987": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ac26758ef636439ea359ddf0d1729743": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "b1f9c517f8404821aa7e374d674982e8": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b20f695e03d748bdb2016a09c6712e6a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_edf5019ce88945bd955f66e539a68c55",
       "IPY_MODEL_e492e498b2da4c3c8be865af31caeac8"
      ],
      "layout": "IPY_MODEL_68c2817c35724d908b2dc27ddbc8d65a"
     }
    },
    "b234ad53c4f048b3b336d9a3e5441189": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "bb6f1ba6e0cf46889984cfeb9bd16097": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "bd7d5597bcc24a5ba6ed233ff6869af3": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c003c5f9ade94009a41a9d225bc12502": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c6c4fa9e09454d6f81d9e7cabc1ad9d8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_e522f0058696468a979a4232787e8057",
       "IPY_MODEL_0e04ea39154b40819a6ae5b5914847a2"
      ],
      "layout": "IPY_MODEL_a845cf55d3cc451cbf6023f53451b987"
     }
    },
    "c76e58b8968747bc915fc37594da12fb": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d0979a4394f84ddb89c4307a18df05ee": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d2077508138b42e688b069d59ba044a0": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d769fc5c617e4a9dbd5ae1b132b9981b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "d8b58951df5c4dfeb6a1d24669b522ff": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "db2ba1f0367d4dc4abcea8e19736a80f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Extraction completed...",
      "description_tooltip": null,
      "layout": "IPY_MODEL_46ec15c44cf0445eb7097b9ffcd48b72",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_d769fc5c617e4a9dbd5ae1b132b9981b",
      "value": 1
     }
    },
    "de36871931e84fef867700db40d0126c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "info",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d0979a4394f84ddb89c4307a18df05ee",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_02fa22f70da849ceb5cef6c6a9a6c42f",
      "value": 1
     }
    },
    "e1050782a98a48c7ba72adacd240b84a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c76e58b8968747bc915fc37594da12fb",
      "placeholder": "​",
      "style": "IPY_MODEL_4469976e9ed0411eb472ce953b2b942f",
      "value": "1/|/100% 1/1 [00:05&lt;00:00,  5.29s/ file]"
     }
    },
    "e44536ead67141a58ce0aaad0ede93d2": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e492e498b2da4c3c8be865af31caeac8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_66669c75d14146139cba84f57a4c12ac",
      "placeholder": "​",
      "style": "IPY_MODEL_f6ca90f14db146c0a44e93674585786d",
      "value": "1/|/100% 1/1 [00:05&lt;00:00,  1.59s/ url]"
     }
    },
    "e522f0058696468a979a4232787e8057": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "danger",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_bd7d5597bcc24a5ba6ed233ff6869af3",
      "max": 51785,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_056d566b9dfc4c12b13b0fcb6075a270",
      "value": 35326
     }
    },
    "e65cf9ab4221419e97f74426d0ee292f": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ea263382231d42b29888fb3d1b16acca": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_6276134a3fa7469faa2cdb9b9841a236",
       "IPY_MODEL_a74e3d4851ad40a9802864c7e9fa3313"
      ],
      "layout": "IPY_MODEL_3dbe4b29ee9d467cbeef39946925db10"
     }
    },
    "eaa4b7905b524f5ea5e2d5c73437e0d5": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "info",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_245051a8a9e746568bc24399fded1713",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_f3d20f2e66ed4f059a3a26ffe703adf3",
      "value": 1
     }
    },
    "ed49da325e7146d5bddbd6b2ee6e338b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_de36871931e84fef867700db40d0126c",
       "IPY_MODEL_2966a7701dde4f46885083de9c6b1eb0"
      ],
      "layout": "IPY_MODEL_2b125be6a55f43798366013f4d4ee840"
     }
    },
    "ed874e7a5d294e28940f3ac0dcc99a96": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "danger",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_5599b46c0a374da4a1bc3566747f395c",
      "max": 1193,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_ac26758ef636439ea359ddf0d1729743",
      "value": 0
     }
    },
    "edf5019ce88945bd955f66e539a68c55": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Dl Completed...",
      "description_tooltip": null,
      "layout": "IPY_MODEL_792f241ea8394e9ca81eeac6909481f8",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_84e50d0a4e824d409249d1c05f13a8fc",
      "value": 1
     }
    },
    "f3d20f2e66ed4f059a3a26ffe703adf3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "f6ca90f14db146c0a44e93674585786d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
